diff --git a/Cargo.lock b/Cargo.lock index 60e2a45ba6..1097ee3fb2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5151,9 +5151,9 @@ dependencies = [ "clap 2.33.3", "log", "rayon", + "solana-accounts-db", "solana-logger", "solana-measure", - "solana-runtime", "solana-sdk", "solana-version", ] @@ -5167,6 +5167,7 @@ dependencies = [ "rand 0.7.3", "rayon", "solana-account-decoder", + "solana-accounts-db", "solana-clap-utils", "solana-cli-config", "solana-client", @@ -5187,6 +5188,84 @@ dependencies = [ "spl-token", ] +[[package]] +name = "solana-accounts-db" +version = "1.17.0" +dependencies = [ + "arrayref", + "assert_matches", + "bincode", + "blake3", + "bv", + "bytemuck", + "byteorder", + "bzip2", + "crossbeam-channel", + "dashmap 4.0.2", + "dir-diff", + "ed25519-dalek", + "flate2", + "fnv", + "fs-err", + "im", + "index_list", + "itertools", + "lazy_static", + "libsecp256k1", + "log", + "lru", + "lz4", + "memmap2", + "memoffset 0.9.0", + "modular-bitfield", + "num-derive", + "num-traits", + "num_cpus", + "num_enum 0.6.1", + "once_cell", + "ouroboros", + "percentage", + "rand 0.7.3", + "rand_chacha 0.2.2", + "rayon", + "regex", + "rustc_version 0.4.0", + "serde", + "serde_derive", + "siphasher", + "solana-accounts-db", + "solana-address-lookup-table-program", + "solana-bpf-loader-program", + "solana-bucket-map", + "solana-compute-budget-program", + "solana-config-program", + "solana-cost-model", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-loader-v4-program", + "solana-logger", + "solana-measure", + "solana-metrics", + "solana-perf", + "solana-program-runtime", + "solana-rayon-threadlimit", + "solana-sdk", + "solana-stake-program", + "solana-system-program", + "solana-vote-program", + "solana-zk-token-proof-program", + "solana-zk-token-sdk", + "static_assertions", + "strum", + "strum_macros", + "symlink", + "tar", + "tempfile", + "test-case", + "thiserror", + "zstd", +] + [[package]] name = "solana-address-lookup-table-program" version = "1.17.0" @@ -5274,6 +5353,7 @@ dependencies = [ "bincode", "crossbeam-channel", "futures 0.3.28", + "solana-accounts-db", "solana-banks-interface", "solana-client", "solana-runtime", @@ -5715,6 +5795,7 @@ dependencies = [ "serde_derive", "serde_json", "serial_test", + "solana-accounts-db", "solana-address-lookup-table-program", "solana-bloom", "solana-client", @@ -5935,6 +6016,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml 0.9.25", + "solana-accounts-db", "solana-clap-utils", "solana-cli-config", "solana-entry", @@ -5953,9 +6035,9 @@ name = "solana-genesis-utils" version = "1.17.0" dependencies = [ "log", + "solana-accounts-db", "solana-download-utils", "solana-rpc-client", - "solana-runtime", "solana-sdk", ] @@ -5981,6 +6063,7 @@ dependencies = [ "libloading", "log", "serde_json", + "solana-accounts-db", "solana-entry", "solana-geyser-plugin-interface", "solana-ledger", @@ -6128,6 +6211,7 @@ dependencies = [ "serde_bytes", "sha2 0.10.7", "solana-account-decoder", + "solana-accounts-db", "solana-bpf-loader-program", "solana-cost-model", "solana-entry", @@ -6181,6 +6265,7 @@ dependencies = [ "serde_json", "signal-hook", "solana-account-decoder", + "solana-accounts-db", "solana-bpf-loader-program", "solana-clap-utils", "solana-cli-output", @@ -6233,6 +6318,7 @@ dependencies = [ "rand 0.7.3", "rayon", "serial_test", + "solana-accounts-db", "solana-client", "solana-config-program", "solana-core", @@ -6295,9 +6381,9 @@ version = "1.17.0" dependencies = [ "clap 2.33.3", "log", + "solana-accounts-db", "solana-logger", "solana-measure", - "solana-runtime", "solana-sdk", "solana-version", ] @@ -6530,6 +6616,7 @@ dependencies = [ "crossbeam-channel", "log", "serde", + "solana-accounts-db", "solana-banks-client", "solana-banks-interface", "solana-banks-server", @@ -6647,6 +6734,7 @@ dependencies = [ "serial_test", "soketto", "solana-account-decoder", + "solana-accounts-db", "solana-address-lookup-table-program", "solana-client", "solana-entry", @@ -6817,6 +6905,7 @@ dependencies = [ "serde", "serde_derive", "siphasher", + "solana-accounts-db", "solana-address-lookup-table-program", "solana-bpf-loader-program", "solana-bucket-map", @@ -7019,8 +7108,8 @@ version = "1.17.0" dependencies = [ "clap 2.33.3", "log", + "solana-accounts-db", "solana-logger", - "solana-runtime", "solana-sdk", "solana-version", ] @@ -7080,6 +7169,7 @@ dependencies = [ "log", "serde_derive", "serde_json", + "solana-accounts-db", "solana-cli-output", "solana-client", "solana-core", @@ -7307,6 +7397,7 @@ dependencies = [ "serde_yaml 0.9.25", "signal-hook", "solana-account-decoder", + "solana-accounts-db", "solana-clap-utils", "solana-cli-config", "solana-core", diff --git a/Cargo.toml b/Cargo.toml index a66c970d15..9aee5ad03e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "account-decoder", "accounts-bench", "accounts-cluster-bench", + "accounts-db", "banking-bench", "banks-client", "banks-interface", @@ -293,6 +294,7 @@ socket2 = "0.5.3" soketto = "0.7" solana_rbpf = "=0.6.0" solana-account-decoder = { path = "account-decoder", version = "=1.17.0" } +solana-accounts-db = { path = "accounts-db", version = "=1.17.0" } solana-address-lookup-table-program = { path = "programs/address-lookup-table", version = "=1.17.0" } solana-banks-client = { path = "banks-client", version = "=1.17.0" } solana-banks-interface = { path = "banks-interface", version = "=1.17.0" } diff --git a/accounts-bench/Cargo.toml b/accounts-bench/Cargo.toml index 3dafb8f8cd..ebc3ac5bed 100644 --- a/accounts-bench/Cargo.toml +++ b/accounts-bench/Cargo.toml @@ -12,9 +12,9 @@ edition = { workspace = true } clap = { workspace = true } log = { workspace = true } rayon = { workspace = true } +solana-accounts-db = { workspace = true } solana-logger = { workspace = true } solana-measure = { workspace = true } -solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-version = { workspace = true } diff --git a/accounts-bench/src/main.rs b/accounts-bench/src/main.rs index 5e208955cb..fdff9ea8b3 100644 --- a/accounts-bench/src/main.rs +++ b/accounts-bench/src/main.rs @@ -5,8 +5,7 @@ extern crate log; use { clap::{crate_description, crate_name, value_t, App, Arg}, rayon::prelude::*, - solana_measure::measure::Measure, - solana_runtime::{ + solana_accounts_db::{ accounts::Accounts, accounts_db::{ test_utils::{create_test_accounts, update_accounts_bench}, @@ -16,6 +15,7 @@ use { ancestors::Ancestors, rent_collector::RentCollector, }, + solana_measure::measure::Measure, solana_sdk::{ genesis_config::ClusterType, pubkey::Pubkey, sysvar::epoch_schedule::EpochSchedule, }, diff --git a/accounts-cluster-bench/Cargo.toml b/accounts-cluster-bench/Cargo.toml index 7d686afda9..8807020d2f 100644 --- a/accounts-cluster-bench/Cargo.toml +++ b/accounts-cluster-bench/Cargo.toml @@ -14,6 +14,7 @@ log = { workspace = true } rand = { workspace = true } rayon = { workspace = true } solana-account-decoder = { workspace = true } +solana-accounts-db = { workspace = true } solana-clap-utils = { workspace = true } solana-cli-config = { workspace = true } solana-client = { workspace = true } diff --git a/accounts-cluster-bench/src/main.rs b/accounts-cluster-bench/src/main.rs index 19c0d8d0e0..7c6ac9d5a8 100644 --- a/accounts-cluster-bench/src/main.rs +++ b/accounts-cluster-bench/src/main.rs @@ -4,6 +4,7 @@ use { log::*, rand::{thread_rng, Rng}, rayon::prelude::*, + solana_accounts_db::inline_spl_token, solana_clap_utils::{ hidden_unless_forced, input_parsers::pubkey_of, input_validators::is_url_or_moniker, }, @@ -11,7 +12,6 @@ use { solana_client::transaction_executor::TransactionExecutor, solana_gossip::gossip_service::discover, solana_rpc_client::rpc_client::RpcClient, - solana_runtime::inline_spl_token, solana_sdk::{ commitment_config::CommitmentConfig, hash::Hash, @@ -703,6 +703,7 @@ fn main() { pub mod test { use { super::*, + solana_accounts_db::inline_spl_token, solana_core::validator::ValidatorConfig, solana_faucet::faucet::run_local_faucet, solana_local_cluster::{ diff --git a/accounts-db/Cargo.toml b/accounts-db/Cargo.toml new file mode 100644 index 0000000000..1bf07989fe --- /dev/null +++ b/accounts-db/Cargo.toml @@ -0,0 +1,101 @@ +[package] +name = "solana-accounts-db" +description = "Solana accounts db" +documentation = "https://docs.rs/solana-acounts-db" +version = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +edition = { workspace = true } + +[dependencies] +arrayref = { workspace = true } +bincode = { workspace = true } +blake3 = { workspace = true } +bv = { workspace = true, features = ["serde"] } +bytemuck = { workspace = true } +byteorder = { workspace = true } +bzip2 = { workspace = true } +crossbeam-channel = { workspace = true } +dashmap = { workspace = true, features = ["rayon", "raw-api"] } +dir-diff = { workspace = true } +flate2 = { workspace = true } +fnv = { workspace = true } +fs-err = { workspace = true } +im = { workspace = true, features = ["rayon", "serde"] } +index_list = { workspace = true } +itertools = { workspace = true } +lazy_static = { workspace = true } +log = { workspace = true } +lru = { workspace = true } +lz4 = { workspace = true } +memmap2 = { workspace = true } +modular-bitfield = { workspace = true } +num-derive = { workspace = true } +num-traits = { workspace = true } +num_cpus = { workspace = true } +num_enum = { workspace = true } +once_cell = { workspace = true } +ouroboros = { workspace = true } +percentage = { workspace = true } +rand = { workspace = true } +rayon = { workspace = true } +regex = { workspace = true } +serde = { workspace = true, features = ["rc"] } +serde_derive = { workspace = true } +siphasher = { workspace = true } +solana-address-lookup-table-program = { workspace = true } +solana-bpf-loader-program = { workspace = true } +solana-bucket-map = { workspace = true } +solana-compute-budget-program = { workspace = true } +solana-config-program = { workspace = true } +solana-cost-model = { workspace = true } +solana-frozen-abi = { workspace = true } +solana-frozen-abi-macro = { workspace = true } +solana-loader-v4-program = { workspace = true } +solana-measure = { workspace = true } +solana-metrics = { workspace = true } +solana-perf = { workspace = true } +solana-program-runtime = { workspace = true } +solana-rayon-threadlimit = { workspace = true } +solana-sdk = { workspace = true } +solana-stake-program = { workspace = true } +solana-system-program = { workspace = true } +solana-vote-program = { workspace = true } +solana-zk-token-proof-program = { workspace = true } +solana-zk-token-sdk = { workspace = true } +static_assertions = { workspace = true } +strum = { workspace = true, features = ["derive"] } +strum_macros = { workspace = true } +symlink = { workspace = true } +tar = { workspace = true } +tempfile = { workspace = true } +thiserror = { workspace = true } +zstd = { workspace = true } + +[lib] +crate-type = ["lib"] +name = "solana_accounts_db" + +[dev-dependencies] +assert_matches = { workspace = true } +ed25519-dalek = { workspace = true } +libsecp256k1 = { workspace = true } +memoffset = { workspace = true } +rand_chacha = { workspace = true } +# See order-crates-for-publishing.py for using this unusual `path = "."` +solana-accounts-db = { path = ".", features = ["dev-context-only-utils"] } +solana-logger = { workspace = true } +solana-sdk = { workspace = true, features = ["dev-context-only-utils"] } +static_assertions = { workspace = true } +test-case = { workspace = true } + +[package.metadata.docs.rs] +targets = ["x86_64-unknown-linux-gnu"] + +[build-dependencies] +rustc_version = { workspace = true } + +[features] +dev-context-only-utils = [] diff --git a/accounts-db/build.rs b/accounts-db/build.rs new file mode 100644 index 0000000000..c9550c1c5c --- /dev/null +++ b/accounts-db/build.rs @@ -0,0 +1,27 @@ +extern crate rustc_version; +use rustc_version::{version_meta, Channel}; + +fn main() { + // Copied and adapted from + // https://github.com/Kimundi/rustc-version-rs/blob/1d692a965f4e48a8cb72e82cda953107c0d22f47/README.md#example + // Licensed under Apache-2.0 + MIT + match version_meta().unwrap().channel { + Channel::Stable => { + println!("cargo:rustc-cfg=RUSTC_WITHOUT_SPECIALIZATION"); + } + Channel::Beta => { + println!("cargo:rustc-cfg=RUSTC_WITHOUT_SPECIALIZATION"); + } + Channel::Nightly => { + println!("cargo:rustc-cfg=RUSTC_WITH_SPECIALIZATION"); + } + Channel::Dev => { + println!("cargo:rustc-cfg=RUSTC_WITH_SPECIALIZATION"); + // See https://github.com/solana-labs/solana/issues/11055 + // We may be running the custom `rust-bpf-builder` toolchain, + // which currently needs `#![feature(proc_macro_hygiene)]` to + // be applied. + println!("cargo:rustc-cfg=RUSTC_NEEDS_PROC_MACRO_HYGIENE"); + } + } +} diff --git a/runtime/src/account_info.rs b/accounts-db/src/account_info.rs similarity index 100% rename from runtime/src/account_info.rs rename to accounts-db/src/account_info.rs diff --git a/runtime/src/account_overrides.rs b/accounts-db/src/account_overrides.rs similarity index 100% rename from runtime/src/account_overrides.rs rename to accounts-db/src/account_overrides.rs diff --git a/runtime/src/account_rent_state.rs b/accounts-db/src/account_rent_state.rs similarity index 97% rename from runtime/src/account_rent_state.rs rename to accounts-db/src/account_rent_state.rs index 284dca2bc3..ce261ec16c 100644 --- a/runtime/src/account_rent_state.rs +++ b/accounts-db/src/account_rent_state.rs @@ -10,7 +10,7 @@ use { }; #[derive(Debug, PartialEq, Eq)] -pub(crate) enum RentState { +pub enum RentState { /// account.lamports == 0 Uninitialized, /// 0 < account.lamports < rent-exempt-minimum @@ -23,7 +23,7 @@ pub(crate) enum RentState { } impl RentState { - pub(crate) fn from_account(account: &AccountSharedData, rent: &Rent) -> Self { + pub fn from_account(account: &AccountSharedData, rent: &Rent) -> Self { if account.lamports() == 0 { Self::Uninitialized } else if rent.is_exempt(account.lamports(), account.data().len()) { @@ -36,7 +36,7 @@ impl RentState { } } - pub(crate) fn transition_allowed_from(&self, pre_rent_state: &RentState) -> bool { + pub fn transition_allowed_from(&self, pre_rent_state: &RentState) -> bool { match self { Self::Uninitialized | Self::RentExempt => true, Self::RentPaying { @@ -73,7 +73,7 @@ pub(crate) fn submit_rent_state_metrics(pre_rent_state: &RentState, post_rent_st } } -pub(crate) fn check_rent_state( +pub fn check_rent_state( pre_rent_state: Option<&RentState>, post_rent_state: Option<&RentState>, transaction_context: &TransactionContext, diff --git a/runtime/src/account_storage.rs b/accounts-db/src/account_storage.rs similarity index 98% rename from runtime/src/account_storage.rs rename to accounts-db/src/account_storage.rs index c3f6563b31..2beebd380a 100644 --- a/runtime/src/account_storage.rs +++ b/accounts-db/src/account_storage.rs @@ -12,10 +12,10 @@ pub mod meta; #[derive(Clone, Debug)] pub struct AccountStorageReference { /// the single storage for a given slot - pub(crate) storage: Arc, + pub storage: Arc, /// id can be read from 'storage', but it is an atomic read. /// id will never change while a storage is held, so we store it separately here for faster runtime lookup in 'get_account_storage_entry' - pub(crate) id: AppendVecId, + pub id: AppendVecId, } pub type AccountStorageMap = DashMap; @@ -74,7 +74,7 @@ impl AccountStorage { /// return the append vec for 'slot' if it exists /// This is only ever called when shrink is not possibly running and there is a max of 1 append vec per slot. - pub(crate) fn get_slot_storage_entry(&self, slot: Slot) -> Option> { + pub fn get_slot_storage_entry(&self, slot: Slot) -> Option> { assert!(self.no_shrink_in_progress()); self.get_slot_storage_entry_shrinking_in_progress_ok(slot) } @@ -100,7 +100,7 @@ impl AccountStorage { } /// initialize the storage map to 'all_storages' - pub(crate) fn initialize(&mut self, all_storages: AccountStorageMap) { + pub fn initialize(&mut self, all_storages: AccountStorageMap) { assert!(self.map.is_empty()); assert!(self.no_shrink_in_progress()); self.map.extend(all_storages.into_iter()) @@ -208,7 +208,7 @@ impl<'a> Iterator for AccountStorageIter<'a> { /// exists while there is a shrink in progress /// keeps track of the 'new_store' being created and the 'old_store' being replaced. #[derive(Debug)] -pub(crate) struct ShrinkInProgress<'a> { +pub struct ShrinkInProgress<'a> { storage: &'a AccountStorage, /// old store which will be shrunk and replaced old_store: Arc, @@ -244,7 +244,7 @@ impl<'a> Drop for ShrinkInProgress<'a> { } impl<'a> ShrinkInProgress<'a> { - pub(crate) fn new_storage(&self) -> &Arc { + pub fn new_storage(&self) -> &Arc { &self.new_store } pub(crate) fn old_storage(&self) -> &Arc { diff --git a/runtime/src/account_storage/meta.rs b/accounts-db/src/account_storage/meta.rs similarity index 100% rename from runtime/src/account_storage/meta.rs rename to accounts-db/src/account_storage/meta.rs diff --git a/runtime/src/accounts.rs b/accounts-db/src/accounts.rs similarity index 99% rename from runtime/src/accounts.rs rename to accounts-db/src/accounts.rs index 5b07d212e4..c2600b8e5a 100644 --- a/runtime/src/accounts.rs +++ b/accounts-db/src/accounts.rs @@ -84,7 +84,7 @@ pub struct AccountLocks { } #[derive(Debug, PartialEq, Eq, Copy, Clone)] -pub(crate) enum RewardInterval { +pub enum RewardInterval { /// the slot within the epoch is INSIDE the reward distribution interval InsideInterval, /// the slot within the epoch is OUTSIDE the reward distribution interval @@ -216,11 +216,11 @@ impl Accounts { )) } - pub(crate) fn new_empty(accounts_db: AccountsDb) -> Self { + pub fn new_empty(accounts_db: AccountsDb) -> Self { Self::new(Arc::new(accounts_db)) } - pub(crate) fn new(accounts_db: Arc) -> Self { + pub fn new(accounts_db: Arc) -> Self { Self { accounts_db, account_locks: Mutex::new(AccountLocks::default()), @@ -694,7 +694,7 @@ impl Accounts { } #[allow(clippy::too_many_arguments)] - pub(crate) fn load_accounts( + pub fn load_accounts( &self, ancestors: &Ancestors, txs: &[SanitizedTransaction], @@ -1306,7 +1306,7 @@ impl Accounts { /// Store the accounts into the DB // allow(clippy) needed for various gating flags #[allow(clippy::too_many_arguments)] - pub(crate) fn store_cached( + pub fn store_cached( &self, slot: Slot, txs: &[SanitizedTransaction], diff --git a/runtime/src/accounts_cache.rs b/accounts-db/src/accounts_cache.rs similarity index 100% rename from runtime/src/accounts_cache.rs rename to accounts-db/src/accounts_cache.rs diff --git a/runtime/src/accounts_db.rs b/accounts-db/src/accounts_db.rs similarity index 98% rename from runtime/src/accounts_db.rs rename to accounts-db/src/accounts_db.rs index d4b8c272f3..6915b007df 100644 --- a/runtime/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -28,12 +28,12 @@ use { }, AccountStorage, AccountStorageStatus, ShrinkInProgress, }, - accounts_background_service::{DroppedSlotsSender, SendDroppedBankCallback}, accounts_cache::{AccountsCache, CachedAccount, SlotCache}, accounts_file::{AccountsFile, AccountsFileError}, accounts_hash::{ AccountsDeltaHash, AccountsHash, AccountsHashEnum, AccountsHasher, CalcAccountsHashConfig, CalculateHashIntermediate, HashStats, IncrementalAccountsHash, + SerdeAccountsDeltaHash, SerdeAccountsHash, SerdeIncrementalAccountsHash, ZeroLamportAccounts, }, accounts_index::{ @@ -62,8 +62,6 @@ use { pubkey_bins::PubkeyBinCalculator24, read_only_accounts_cache::ReadOnlyAccountsCache, rent_collector::RentCollector, - serde_snapshot::{SerdeAccountsDeltaHash, SerdeAccountsHash, SerdeIncrementalAccountsHash}, - snapshot_utils::create_accounts_run_and_snapshot_dirs, sorted_storages::SortedStorages, storable_accounts::StorableAccounts, verify_accounts_hash_in_background::VerifyAccountsHashInBackground, @@ -394,7 +392,7 @@ enum LoadZeroLamports { /// Note that this is non-deterministic if clean is running asynchronously. /// If a zero lamport account exists in the index, then Some is returned. /// Once it is cleaned from the index, None is returned. - #[cfg(test)] + #[cfg(feature = "dev-context-only-utils")] SomeWithZeroLamportAccountForTests, } @@ -1021,7 +1019,7 @@ pub struct AccountStorageEntry { pub(crate) slot: AtomicU64, /// storage holding the accounts - pub(crate) accounts: AccountsFile, + pub accounts: AccountsFile, /// Keeps track of the number of accounts stored in a specific AppendVec. /// This is periodically checked to reuse the stores that do not have @@ -1057,7 +1055,7 @@ impl AccountStorageEntry { } } - pub(crate) fn new_existing( + pub fn new_existing( slot: Slot, id: AppendVecId, accounts: AccountsFile, @@ -1211,6 +1209,76 @@ impl AccountStorageEntry { } } +/// To allow generating a bank snapshot directory with full state information, we need to +/// hardlink account appendvec files from the runtime operation directory to a snapshot +/// hardlink directory. This is to create the run/ and snapshot sub directories for an +/// account_path provided by the user. These two sub directories are on the same file +/// system partition to allow hard-linking. +pub fn create_accounts_run_and_snapshot_dirs( + account_dir: impl AsRef, +) -> std::io::Result<(PathBuf, PathBuf)> { + let run_path = account_dir.as_ref().join("run"); + let snapshot_path = account_dir.as_ref().join("snapshot"); + if (!run_path.is_dir()) || (!snapshot_path.is_dir()) { + // If the "run/" or "snapshot" sub directories do not exist, the directory may be from + // an older version for which the appendvec files are at this directory. Clean up + // them first. + // This will be done only once when transitioning from an old image without run directory + // to this new version using run and snapshot directories. + // The run/ content cleanup will be done at a later point. The snapshot/ content persists + // across the process boot, and will be purged by the account_background_service. + if fs_err::remove_dir_all(&account_dir).is_err() { + delete_contents_of_path(&account_dir); + } + fs_err::create_dir_all(&run_path)?; + fs_err::create_dir_all(&snapshot_path)?; + } + + Ok((run_path, snapshot_path)) +} + +/// For all account_paths, create the run/ and snapshot/ sub directories. +/// If an account_path directory does not exist, create it. +/// It returns (account_run_paths, account_snapshot_paths) or error +pub fn create_all_accounts_run_and_snapshot_dirs( + account_paths: &[PathBuf], +) -> std::io::Result<(Vec, Vec)> { + let mut run_dirs = Vec::with_capacity(account_paths.len()); + let mut snapshot_dirs = Vec::with_capacity(account_paths.len()); + for account_path in account_paths { + // create the run/ and snapshot/ sub directories for each account_path + let (run_dir, snapshot_dir) = create_accounts_run_and_snapshot_dirs(account_path)?; + run_dirs.push(run_dir); + snapshot_dirs.push(snapshot_dir); + } + Ok((run_dirs, snapshot_dirs)) +} + +/// Delete the files and subdirectories in a directory. +/// This is useful if the process does not have permission +/// to delete the top level directory it might be able to +/// delete the contents of that directory. +pub fn delete_contents_of_path(path: impl AsRef) { + match fs_err::read_dir(path.as_ref()) { + Err(err) => { + warn!("Failed to delete contents: {err}") + } + Ok(dir_entries) => { + for entry in dir_entries.flatten() { + let sub_path = entry.path(); + let result = if sub_path.is_dir() { + fs_err::remove_dir_all(&sub_path) + } else { + fs_err::remove_file(&sub_path) + }; + if let Err(err) = result { + warn!("Failed to delete contents: {err}"); + } + } + } + } +} + pub fn get_temp_accounts_paths(count: u32) -> IoResult<(Vec, Vec)> { let temp_dirs: IoResult> = (0..count).map(|_| TempDir::new()).collect(); let temp_dirs = temp_dirs?; @@ -1371,7 +1439,7 @@ pub struct AccountsDb { /// true iff we want to skip the initial hash calculation on startup pub skip_initial_hash_calc: bool, - pub(crate) storage: AccountStorage, + pub storage: AccountStorage, #[allow(dead_code)] /// from AccountsDbConfig @@ -1392,10 +1460,10 @@ pub struct AccountsDb { /// Set of shrinkable stores organized by map of slot to append_vec_id pub shrink_candidate_slots: Mutex, - pub(crate) write_version: AtomicU64, + pub write_version: AtomicU64, /// Set of storage paths to pick from - pub(crate) paths: Vec, + pub paths: Vec, full_accounts_hash_cache_path: PathBuf, incremental_accounts_hash_cache_path: PathBuf, @@ -1410,7 +1478,7 @@ pub struct AccountsDb { /// Directory of paths this accounts_db needs to hold/remove #[allow(dead_code)] - pub(crate) temp_paths: Option>, + pub temp_paths: Option>, /// Starting file size of appendvecs file_size: u64, @@ -1433,7 +1501,7 @@ pub struct AccountsDb { // Stats for purges called outside of clean_accounts() external_purge_slots_stats: PurgeStats, - pub(crate) shrink_stats: ShrinkStats, + pub shrink_stats: ShrinkStats, pub(crate) shrink_ancient_stats: ShrinkAncientStats, @@ -1485,18 +1553,18 @@ pub struct AccountsDb { /// number of slots remaining where filler accounts should be added pub filler_account_slots_remaining: AtomicU64, - pub(crate) verify_accounts_hash_in_bg: VerifyAccountsHashInBackground, + pub verify_accounts_hash_in_bg: VerifyAccountsHashInBackground, /// Used to disable logging dead slots during removal. /// allow disabling noisy log - pub(crate) log_dead_slots: AtomicBool, + pub log_dead_slots: AtomicBool, /// debug feature to scan every append vec and verify refcounts are equal exhaustively_verify_refcounts: bool, /// this will live here until the feature for partitioned epoch rewards is activated. /// At that point, this and other code can be deleted. - pub(crate) partitioned_epoch_rewards_config: PartitionedEpochRewardsConfig, + pub partitioned_epoch_rewards_config: PartitionedEpochRewardsConfig, /// the full accounts hash calculation as of a predetermined block height 'N' /// to be included in the bank hash at a predetermined block height 'M' @@ -1534,7 +1602,7 @@ pub struct AccountsStats { } #[derive(Debug, Default)] -pub(crate) struct PurgeStats { +pub struct PurgeStats { last_report: AtomicInterval, safety_checks_elapsed: AtomicU64, remove_cache_elapsed: AtomicU64, @@ -1949,7 +2017,7 @@ impl ShrinkStatsSub { } #[derive(Debug, Default)] -pub(crate) struct ShrinkStats { +pub struct ShrinkStats { last_report: AtomicInterval, num_slots_shrunk: AtomicUsize, storage_read_elapsed: AtomicU64, @@ -2233,7 +2301,7 @@ pub fn make_min_priority_thread_pool() -> ThreadPool { .unwrap() } -#[cfg(all(test, RUSTC_WITH_SPECIALIZATION))] +#[cfg(RUSTC_WITH_SPECIALIZATION)] impl solana_frozen_abi::abi_example::AbiExample for AccountsDb { fn example() -> Self { let accounts_db = AccountsDb::new_single_for_tests(); @@ -2834,7 +2902,7 @@ impl AccountsDb { } #[must_use] - pub(crate) fn purge_keys_exact<'a, C: 'a>( + pub fn purge_keys_exact<'a, C: 'a>( &'a self, pubkey_to_slot_set: impl Iterator, ) -> (Vec<(Slot, AccountInfo)>, PubkeysRemovedFromAccountsIndex) @@ -3805,7 +3873,7 @@ impl AccountsDb { /// get all accounts in all the storages passed in /// for duplicate pubkeys, the account with the highest write_value is returned - pub(crate) fn get_unique_accounts_from_storage<'a>( + pub fn get_unique_accounts_from_storage<'a>( &self, store: &'a Arc, ) -> GetUniqueAccountsResult<'a> { @@ -4084,7 +4152,7 @@ impl AccountsDb { /// Drop 'shrink_in_progress', which will cause the old store to be removed from the storage map. /// For 'shrink_in_progress'.'old_storage' which is not retained, insert in 'dead_storages' and optionally 'dirty_stores' /// This is the end of the life cycle of `shrink_in_progress`. - pub(crate) fn mark_dirty_dead_stores( + pub fn mark_dirty_dead_stores( &self, slot: Slot, add_dirty_stores: bool, @@ -4112,7 +4180,7 @@ impl AccountsDb { dead_storages } - pub(crate) fn drop_or_recycle_stores( + pub fn drop_or_recycle_stores( &self, dead_storages: Vec>, stats: &ShrinkStats, @@ -4142,11 +4210,7 @@ impl AccountsDb { } /// return a store that can contain 'aligned_total' bytes - pub(crate) fn get_store_for_shrink( - &self, - slot: Slot, - aligned_total: u64, - ) -> ShrinkInProgress<'_> { + pub fn get_store_for_shrink(&self, slot: Slot, aligned_total: u64) -> ShrinkInProgress<'_> { let shrunken_store = self .try_recycle_store(slot, aligned_total, aligned_total + 1024) .unwrap_or_else(|| { @@ -4316,20 +4380,6 @@ impl AccountsDb { } } - #[cfg(test)] - pub(crate) fn sizes_of_accounts_in_storage_for_tests(&self, slot: Slot) -> Vec { - self.storage - .get_slot_storage_entry(slot) - .map(|storage| { - storage - .accounts - .account_iter() - .map(|account| account.stored_size()) - .collect() - }) - .unwrap_or_default() - } - /// 'accounts' that exist in the current slot we are combining into a different ancient slot /// 'existing_ancient_pubkeys': pubkeys that exist currently in the ancient append vec slot /// returns the pubkeys that are in 'accounts' that are already in 'existing_ancient_pubkeys' @@ -4964,7 +5014,7 @@ impl AccountsDb { /// Insert a default bank hash stats for `slot` /// /// This fn is called when creating a new bank from parent. - pub(crate) fn insert_default_bank_hash_stats(&self, slot: Slot, parent_slot: Slot) { + pub fn insert_default_bank_hash_stats(&self, slot: Slot, parent_slot: Slot) { let mut bank_hash_stats = self.bank_hash_stats.lock().unwrap(); if bank_hash_stats.get(&slot).is_some() { error!("set_hash: already exists; multiple forks with shared slot {slot} as child (parent: {parent_slot})!?"); @@ -5613,7 +5663,7 @@ impl AccountsDb { store } - pub(crate) fn page_align(size: u64) -> u64 { + pub fn page_align(size: u64) -> u64 { (size + (PAGE_SIZE - 1)) & !(PAGE_SIZE - 1) } @@ -5684,13 +5734,9 @@ impl AccountsDb { self.storage.insert(slot, store) } - pub fn create_drop_bank_callback( - &self, - pruned_banks_sender: DroppedSlotsSender, - ) -> SendDroppedBankCallback { + pub fn enable_bank_drop_callback(&self) { self.is_bank_drop_callback_enabled .store(true, Ordering::Release); - SendDroppedBankCallback::new(pruned_banks_sender) } /// This should only be called after the `Bank::drop()` runs in bank.rs, See BANK_DROP_SAFETY @@ -5748,7 +5794,7 @@ impl AccountsDb { /// Purges every slot in `removed_slots` from both the cache and storage. This includes /// entries in the accounts index, cache entries, and any backing storage entries. - pub(crate) fn purge_slots_from_cache_and_store<'a>( + pub fn purge_slots_from_cache_and_store<'a>( &self, removed_slots: impl Iterator + Clone, purge_stats: &PurgeStats, @@ -6248,8 +6294,9 @@ impl AccountsDb { .fetch_add(recycle_stores_write_elapsed.as_us(), Ordering::Relaxed); } - #[cfg(test)] - pub(crate) fn flush_accounts_cache_slot_for_tests(&self, slot: Slot) { + // These functions/fields are only usable from a dev context (i.e. tests and benches) + #[cfg(feature = "dev-context-only-utils")] + pub fn flush_accounts_cache_slot_for_tests(&self, slot: Slot) { self.flush_slot_cache(slot); } @@ -6567,7 +6614,7 @@ impl AccountsDb { /// However, there is a clear path to be able to support this. /// So, combine all accounts from 'slot_stores' into a new storage and return it. /// This runs prior to the storages being put in AccountsDb.storage - pub(crate) fn combine_multiple_slots_into_one_at_startup( + pub fn combine_multiple_slots_into_one_at_startup( path: &Path, id: AppendVecId, slot: Slot, @@ -7262,7 +7309,7 @@ impl AccountsDb { stats.num_dirty_slots = num_dirty_slots; } - pub(crate) fn calculate_accounts_hash( + pub fn calculate_accounts_hash( &self, data_source: CalcAccountsHashDataSource, slot: Slot, @@ -7781,10 +7828,7 @@ impl AccountsDb { /// 1. pubkey, hash pairs for the slot /// 2. us spent scanning /// 3. Measure started when we began accumulating - pub(crate) fn get_pubkey_hash_for_slot( - &self, - slot: Slot, - ) -> (Vec<(Pubkey, Hash)>, u64, Measure) { + pub fn get_pubkey_hash_for_slot(&self, slot: Slot) -> (Vec<(Pubkey, Hash)>, u64, Measure) { let mut scan = Measure::start("scan"); let scan_result: ScanStorageResult<(Pubkey, Hash), DashMap> = self @@ -7821,7 +7865,7 @@ impl AccountsDb { /// /// As part of calculating the accounts delta hash, get a list of accounts modified this slot /// (aka dirty pubkeys) and add them to `self.uncleaned_pubkeys` for future cleaning. - pub(crate) fn calculate_accounts_delta_hash_internal( + pub fn calculate_accounts_delta_hash_internal( &self, slot: Slot, ignore: Option, @@ -8569,7 +8613,7 @@ impl AccountsDb { ); } - pub(crate) fn store_accounts_frozen<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>( + pub fn store_accounts_frozen<'a, T: ReadableAccount + Sync + ZeroLamport + 'a>( &self, accounts: impl StorableAccounts<'a, T>, hashes: Option>>, @@ -9381,7 +9425,7 @@ impl AccountsDb { timings.storage_size_storages_us = storage_size_storages_time.as_us(); } - pub(crate) fn print_accounts_stats(&self, label: &str) { + pub fn print_accounts_stats(&self, label: &str) { self.print_index(label); self.print_count_and_status(label); info!("recycle_stores:"); @@ -9474,10 +9518,6 @@ pub(crate) enum UpdateIndexThreadSelection { #[cfg(test)] impl AccountsDb { - pub fn new(paths: Vec, cluster_type: &ClusterType) -> Self { - Self::new_for_tests(paths, cluster_type) - } - pub fn new_with_config_for_tests( paths: Vec, cluster_type: &ClusterType, @@ -9521,24 +9561,30 @@ impl AccountsDb { let result = self.accounts_index.get(pubkey, Some(&ancestors), None); result.map(|(list, index)| list.slot_list()[index].1.store_id()) } - - pub fn alive_account_count_in_slot(&self, slot: Slot) -> usize { - self.storage - .get_slot_storage_entry(slot) - .map(|storage| storage.count()) - .unwrap_or(0) - .saturating_add( - self.accounts_cache - .slot_cache(slot) - .map(|slot_cache| slot_cache.len()) - .unwrap_or_default(), - ) - } } // These functions/fields are only usable from a dev context (i.e. tests and benches) #[cfg(feature = "dev-context-only-utils")] impl AccountsDb { + pub fn new(paths: Vec, cluster_type: &ClusterType) -> Self { + Self::new_for_tests(paths, cluster_type) + } + + pub fn load_without_fixed_root( + &self, + ancestors: &Ancestors, + pubkey: &Pubkey, + ) -> Option<(AccountSharedData, Slot)> { + self.do_load( + ancestors, + pubkey, + None, + LoadHint::Unspecified, + // callers of this expect zero lamport accounts that exist in the index to be returned as Some(empty) + LoadZeroLamports::SomeWithZeroLamportAccountForTests, + ) + } + pub fn accounts_delta_hashes(&self) -> &Mutex> { &self.accounts_delta_hashes } @@ -9558,6 +9604,182 @@ impl AccountsDb { pub fn set_accounts_hash_for_tests(&self, slot: Slot, accounts_hash: AccountsHash) { self.set_accounts_hash(slot, (accounts_hash, u64::default())); } + + pub fn assert_load_account(&self, slot: Slot, pubkey: Pubkey, expected_lamports: u64) { + let ancestors = vec![(slot, 0)].into_iter().collect(); + let (account, slot) = self.load_without_fixed_root(&ancestors, &pubkey).unwrap(); + assert_eq!((account.lamports(), slot), (expected_lamports, slot)); + } + + pub fn assert_not_load_account(&self, slot: Slot, pubkey: Pubkey) { + let ancestors = vec![(slot, 0)].into_iter().collect(); + let load = self.load_without_fixed_root(&ancestors, &pubkey); + assert!(load.is_none(), "{load:?}"); + } + + pub fn check_accounts(&self, pubkeys: &[Pubkey], slot: Slot, num: usize, count: usize) { + let ancestors = vec![(slot, 0)].into_iter().collect(); + for _ in 0..num { + let idx = thread_rng().gen_range(0, num); + let account = self.load_without_fixed_root(&ancestors, &pubkeys[idx]); + let account1 = Some(( + AccountSharedData::new( + (idx + count) as u64, + 0, + AccountSharedData::default().owner(), + ), + slot, + )); + assert_eq!(account, account1); + } + } + + /// callers used to call store_uncached. But, this is not allowed anymore. + pub fn store_for_tests(&self, slot: Slot, accounts: &[(&Pubkey, &AccountSharedData)]) { + self.store( + (slot, accounts, INCLUDE_SLOT_IN_HASH_TESTS), + &StoreTo::Cache, + None, + StoreReclaims::Default, + UpdateIndexThreadSelection::PoolWithThreshold, + ); + } + + #[allow(clippy::needless_range_loop)] + pub fn modify_accounts(&self, pubkeys: &[Pubkey], slot: Slot, num: usize, count: usize) { + for idx in 0..num { + let account = AccountSharedData::new( + (idx + count) as u64, + 0, + AccountSharedData::default().owner(), + ); + self.store_for_tests(slot, &[(&pubkeys[idx], &account)]); + } + } + + pub fn check_storage(&self, slot: Slot, count: usize) { + assert!(self.storage.get_slot_storage_entry(slot).is_some()); + let store = self.storage.get_slot_storage_entry(slot).unwrap(); + let total_count = store.count(); + assert_eq!(store.status(), AccountStorageStatus::Available); + assert_eq!(total_count, count); + let (expected_store_count, actual_store_count): (usize, usize) = + (store.approx_stored_count(), store.all_accounts().len()); + assert_eq!(expected_store_count, actual_store_count); + } + + pub fn create_account( + &self, + pubkeys: &mut Vec, + slot: Slot, + num: usize, + space: usize, + num_vote: usize, + ) { + let ancestors = vec![(slot, 0)].into_iter().collect(); + for t in 0..num { + let pubkey = solana_sdk::pubkey::new_rand(); + let account = + AccountSharedData::new((t + 1) as u64, space, AccountSharedData::default().owner()); + pubkeys.push(pubkey); + assert!(self.load_without_fixed_root(&ancestors, &pubkey).is_none()); + self.store_for_tests(slot, &[(&pubkey, &account)]); + } + for t in 0..num_vote { + let pubkey = solana_sdk::pubkey::new_rand(); + let account = + AccountSharedData::new((num + t + 1) as u64, space, &solana_vote_program::id()); + pubkeys.push(pubkey); + let ancestors = vec![(slot, 0)].into_iter().collect(); + assert!(self.load_without_fixed_root(&ancestors, &pubkey).is_none()); + self.store_for_tests(slot, &[(&pubkey, &account)]); + } + } + + pub fn sizes_of_accounts_in_storage_for_tests(&self, slot: Slot) -> Vec { + self.storage + .get_slot_storage_entry(slot) + .map(|storage| { + storage + .accounts + .account_iter() + .map(|account| account.stored_size()) + .collect() + }) + .unwrap_or_default() + } + + pub fn ref_count_for_pubkey(&self, pubkey: &Pubkey) -> RefCount { + self.accounts_index.ref_count_from_storage(pubkey) + } + + pub fn alive_account_count_in_slot(&self, slot: Slot) -> usize { + self.storage + .get_slot_storage_entry(slot) + .map(|storage| storage.count()) + .unwrap_or(0) + .saturating_add( + self.accounts_cache + .slot_cache(slot) + .map(|slot_cache| slot_cache.len()) + .unwrap_or_default(), + ) + } + + /// useful to adapt tests written prior to introduction of the write cache + /// to use the write cache + pub fn add_root_and_flush_write_cache(&self, slot: Slot) { + self.add_root(slot); + self.flush_root_write_cache(slot); + } + + /// useful to adapt tests written prior to introduction of the write cache + /// to use the write cache + pub fn flush_root_write_cache(&self, root: Slot) { + assert!( + self.accounts_index + .roots_tracker + .read() + .unwrap() + .alive_roots + .contains(&root), + "slot: {root}" + ); + self.flush_accounts_cache(true, Some(root)); + } + + pub fn all_account_count_in_append_vec(&self, slot: Slot) -> usize { + let store = self.storage.get_slot_storage_entry(slot); + if let Some(store) = store { + let count = store.all_accounts().len(); + let stored_count = store.approx_stored_count(); + assert_eq!(stored_count, count); + count + } else { + 0 + } + } +} + +// These functions/fields are only usable from a dev context (i.e. tests and benches) +#[cfg(feature = "dev-context-only-utils")] +impl<'a> VerifyAccountsHashAndLamportsConfig<'a> { + pub fn new_for_test( + ancestors: &'a Ancestors, + epoch_schedule: &'a EpochSchedule, + rent_collector: &'a RentCollector, + ) -> Self { + Self { + ancestors, + test_hash_calculation: true, + epoch_schedule, + rent_collector, + ignore_mismatch: false, + store_detailed_debug_info: false, + use_bg_thread_pool: false, + include_slot_in_hash: INCLUDE_SLOT_IN_HASH_TESTS, + } + } } /// A set of utility functions used for testing and benchmarking @@ -9649,7 +9871,7 @@ pub mod tests { }, }; - pub fn linear_ancestors(end_slot: u64) -> Ancestors { + fn linear_ancestors(end_slot: u64) -> Ancestors { let mut ancestors: Ancestors = vec![(0, 0)].into_iter().collect(); for i in 1..end_slot { ancestors.insert(i, (i - 1) as usize); @@ -9688,21 +9910,6 @@ pub mod tests { ) } - pub fn load_without_fixed_root( - &self, - ancestors: &Ancestors, - pubkey: &Pubkey, - ) -> Option<(AccountSharedData, Slot)> { - self.do_load( - ancestors, - pubkey, - None, - LoadHint::Unspecified, - // callers of this expect zero lamport accounts that exist in the index to be returned as Some(empty) - LoadZeroLamports::SomeWithZeroLamportAccountForTests, - ) - } - fn get_storage_for_slot(&self, slot: Slot) -> Option> { self.storage.get_slot_storage_entry(slot) } @@ -9776,25 +9983,6 @@ pub mod tests { } } - impl<'a> VerifyAccountsHashAndLamportsConfig<'a> { - pub fn new_for_test( - ancestors: &'a Ancestors, - epoch_schedule: &'a EpochSchedule, - rent_collector: &'a RentCollector, - ) -> Self { - Self { - ancestors, - test_hash_calculation: true, - epoch_schedule, - rent_collector, - ignore_mismatch: false, - store_detailed_debug_info: false, - use_bg_thread_pool: false, - include_slot_in_hash: INCLUDE_SLOT_IN_HASH_TESTS, - } - } - } - #[test] fn test_maybe_unref_accounts_already_in_ancient() { let db = AccountsDb::new_single_for_tests(); @@ -11151,7 +11339,7 @@ pub mod tests { let db = AccountsDb::new(Vec::new(), &ClusterType::Development); let mut pubkeys: Vec = vec![]; - create_account(&db, &mut pubkeys, 0, 100, 0, 0); + db.create_account(&mut pubkeys, 0, 100, 0, 0); for _ in 1..100 { let idx = thread_rng().gen_range(0, 99); let ancestors = vec![(0, 0)].into_iter().collect(); @@ -11193,9 +11381,9 @@ pub mod tests { let db = AccountsDb::new_single_for_tests(); let mut pubkeys: Vec = vec![]; - create_account(&db, &mut pubkeys, 0, 2, DEFAULT_FILE_SIZE as usize / 3, 0); + db.create_account(&mut pubkeys, 0, 2, DEFAULT_FILE_SIZE as usize / 3, 0); db.add_root_and_flush_write_cache(0); - check_storage(&db, 0, 2); + db.check_storage(0, 2); let pubkey = solana_sdk::pubkey::new_rand(); let account = AccountSharedData::new(1, DEFAULT_FILE_SIZE as usize / 3, &pubkey); @@ -11272,7 +11460,7 @@ pub mod tests { .accounts_index .get(&key, Some(&ancestors), None) .is_some()); - assert_load_account(&db, unrooted_slot, key, 1); + db.assert_load_account(unrooted_slot, key, 1); // Purge the slot db.remove_unrooted_slots(&[(unrooted_slot, unrooted_bank_id)]); @@ -11289,7 +11477,7 @@ pub mod tests { // Test we can store for the same slot again and get the right information let account0 = AccountSharedData::new(2, 0, &key); db.store_for_tests(unrooted_slot, &[(&key, &account0)]); - assert_load_account(&db, unrooted_slot, key, 2); + db.assert_load_account(unrooted_slot, key, 2); } #[test] @@ -11302,38 +11490,6 @@ pub mod tests { run_test_remove_unrooted_slot(false); } - pub fn create_account( - accounts: &AccountsDb, - pubkeys: &mut Vec, - slot: Slot, - num: usize, - space: usize, - num_vote: usize, - ) { - let ancestors = vec![(slot, 0)].into_iter().collect(); - for t in 0..num { - let pubkey = solana_sdk::pubkey::new_rand(); - let account = - AccountSharedData::new((t + 1) as u64, space, AccountSharedData::default().owner()); - pubkeys.push(pubkey); - assert!(accounts - .load_without_fixed_root(&ancestors, &pubkey) - .is_none()); - accounts.store_for_tests(slot, &[(&pubkey, &account)]); - } - for t in 0..num_vote { - let pubkey = solana_sdk::pubkey::new_rand(); - let account = - AccountSharedData::new((num + t + 1) as u64, space, &solana_vote_program::id()); - pubkeys.push(pubkey); - let ancestors = vec![(slot, 0)].into_iter().collect(); - assert!(accounts - .load_without_fixed_root(&ancestors, &pubkey) - .is_none()); - accounts.store_for_tests(slot, &[(&pubkey, &account)]); - } - } - fn update_accounts(accounts: &AccountsDb, pubkeys: &[Pubkey], slot: Slot, range: usize) { for _ in 1..1000 { let idx = thread_rng().gen_range(0, range); @@ -11359,64 +11515,12 @@ pub mod tests { } } - pub fn check_storage(accounts: &AccountsDb, slot: Slot, count: usize) { - assert!(accounts.storage.get_slot_storage_entry(slot).is_some()); - let store = accounts.storage.get_slot_storage_entry(slot).unwrap(); - let total_count = store.count(); - assert_eq!(store.status(), AccountStorageStatus::Available); - assert_eq!(total_count, count); - let (expected_store_count, actual_store_count): (usize, usize) = - (store.approx_stored_count(), store.all_accounts().len()); - assert_eq!(expected_store_count, actual_store_count); - } - - pub fn check_accounts( - accounts: &AccountsDb, - pubkeys: &[Pubkey], - slot: Slot, - num: usize, - count: usize, - ) { - let ancestors = vec![(slot, 0)].into_iter().collect(); - for _ in 0..num { - let idx = thread_rng().gen_range(0, num); - let account = accounts.load_without_fixed_root(&ancestors, &pubkeys[idx]); - let account1 = Some(( - AccountSharedData::new( - (idx + count) as u64, - 0, - AccountSharedData::default().owner(), - ), - slot, - )); - assert_eq!(account, account1); - } - } - - #[allow(clippy::needless_range_loop)] - pub fn modify_accounts( - accounts: &AccountsDb, - pubkeys: &[Pubkey], - slot: Slot, - num: usize, - count: usize, - ) { - for idx in 0..num { - let account = AccountSharedData::new( - (idx + count) as u64, - 0, - AccountSharedData::default().owner(), - ); - accounts.store_for_tests(slot, &[(&pubkeys[idx], &account)]); - } - } - #[test] fn test_account_one() { let (_accounts_dirs, paths) = get_temp_accounts_paths(1).unwrap(); let db = AccountsDb::new(paths, &ClusterType::Development); let mut pubkeys: Vec = vec![]; - create_account(&db, &mut pubkeys, 0, 1, 0, 0); + db.create_account(&mut pubkeys, 0, 1, 0, 0); let ancestors = vec![(0, 0)].into_iter().collect(); let account = db.load_without_fixed_root(&ancestors, &pubkeys[0]).unwrap(); let default_account = AccountSharedData::from(Account { @@ -11431,18 +11535,18 @@ pub mod tests { let (_accounts_dirs, paths) = get_temp_accounts_paths(2).unwrap(); let db = AccountsDb::new(paths, &ClusterType::Development); let mut pubkeys: Vec = vec![]; - create_account(&db, &mut pubkeys, 0, 100, 0, 0); - check_accounts(&db, &pubkeys, 0, 100, 1); + db.create_account(&mut pubkeys, 0, 100, 0, 0); + db.check_accounts(&pubkeys, 0, 100, 1); } #[test] fn test_account_update() { let accounts = AccountsDb::new_single_for_tests(); let mut pubkeys: Vec = vec![]; - create_account(&accounts, &mut pubkeys, 0, 100, 0, 0); + accounts.create_account(&mut pubkeys, 0, 100, 0, 0); update_accounts(&accounts, &pubkeys, 0, 99); accounts.add_root_and_flush_write_cache(0); - check_storage(&accounts, 0, 100); + accounts.check_storage(0, 100); } #[test] @@ -11613,24 +11717,6 @@ pub mod tests { ); } - impl AccountsDb { - pub fn all_account_count_in_append_vec(&self, slot: Slot) -> usize { - let store = self.storage.get_slot_storage_entry(slot); - if let Some(store) = store { - let count = store.all_accounts().len(); - let stored_count = store.approx_stored_count(); - assert_eq!(stored_count, count); - count - } else { - 0 - } - } - - pub fn ref_count_for_pubkey(&self, pubkey: &Pubkey) -> RefCount { - self.accounts_index.ref_count_from_storage(pubkey) - } - } - #[test] fn test_clean_zero_lamport_and_dead_slot() { solana_logger::setup(); @@ -12075,25 +12161,6 @@ pub mod tests { assert_eq!(accounts.accounts_index.uncleaned_roots_len(), 0); } - pub fn assert_load_account( - accounts: &AccountsDb, - slot: Slot, - pubkey: Pubkey, - expected_lamports: u64, - ) { - let ancestors = vec![(slot, 0)].into_iter().collect(); - let (account, slot) = accounts - .load_without_fixed_root(&ancestors, &pubkey) - .unwrap(); - assert_eq!((account.lamports(), slot), (expected_lamports, slot)); - } - - pub fn assert_not_load_account(accounts: &AccountsDb, slot: Slot, pubkey: Pubkey) { - let ancestors = vec![(slot, 0)].into_iter().collect(); - let load = accounts.load_without_fixed_root(&ancestors, &pubkey); - assert!(load.is_none(), "{load:?}"); - } - fn assert_no_stores(accounts: &AccountsDb, slot: Slot) { let store = accounts.storage.get_slot_storage_entry(slot); assert!(store.is_none()); @@ -12148,7 +12215,7 @@ pub mod tests { accounts.calculate_accounts_delta_hash(current_slot); accounts.add_root_and_flush_write_cache(current_slot); - assert_load_account(&accounts, current_slot, pubkey, zero_lamport); + accounts.assert_load_account(current_slot, pubkey, zero_lamport); current_slot += 1; accounts.calculate_accounts_delta_hash(current_slot); @@ -12177,9 +12244,9 @@ pub mod tests { // storage for slot 1 had 2 accounts, now has 1 after pubkey 1 // was reclaimed - check_storage(&accounts, 1, 1); + accounts.check_storage(1, 1); // storage for slot 2 had 1 accounts, now has 1 - check_storage(&accounts, 2, 1); + accounts.check_storage(2, 1); } #[test] @@ -12208,7 +12275,7 @@ pub mod tests { accounts.calculate_accounts_delta_hash(current_slot); accounts.add_root_and_flush_write_cache(current_slot); - assert_load_account(&accounts, current_slot, pubkey, zero_lamport); + accounts.assert_load_account(current_slot, pubkey, zero_lamport); // Otherwise slot 2 will not be removed current_slot += 1; @@ -15970,39 +16037,6 @@ pub mod tests { } impl AccountsDb { - /// useful to adapt tests written prior to introduction of the write cache - /// to use the write cache - pub fn add_root_and_flush_write_cache(&self, slot: Slot) { - self.add_root(slot); - self.flush_root_write_cache(slot); - } - - /// useful to adapt tests written prior to introduction of the write cache - /// to use the write cache - pub(crate) fn flush_root_write_cache(&self, root: Slot) { - assert!( - self.accounts_index - .roots_tracker - .read() - .unwrap() - .alive_roots - .contains(&root), - "slot: {root}" - ); - self.flush_accounts_cache(true, Some(root)); - } - - /// callers used to call store_uncached. But, this is not allowed anymore. - pub fn store_for_tests(&self, slot: Slot, accounts: &[(&Pubkey, &AccountSharedData)]) { - self.store( - (slot, accounts, INCLUDE_SLOT_IN_HASH_TESTS), - &StoreTo::Cache, - None, - StoreReclaims::Default, - UpdateIndexThreadSelection::PoolWithThreshold, - ); - } - /// helper function to test unref_accounts or clean_dead_slots_from_accounts_index fn test_unref( &self, diff --git a/runtime/src/accounts_db/geyser_plugin_utils.rs b/accounts-db/src/accounts_db/geyser_plugin_utils.rs similarity index 100% rename from runtime/src/accounts_db/geyser_plugin_utils.rs rename to accounts-db/src/accounts_db/geyser_plugin_utils.rs diff --git a/runtime/src/accounts_file.rs b/accounts-db/src/accounts_file.rs similarity index 100% rename from runtime/src/accounts_file.rs rename to accounts-db/src/accounts_file.rs diff --git a/runtime/src/accounts_hash.rs b/accounts-db/src/accounts_hash.rs similarity index 98% rename from runtime/src/accounts_hash.rs rename to accounts-db/src/accounts_hash.rs index ccbcee8abb..8e27b29b75 100644 --- a/runtime/src/accounts_hash.rs +++ b/accounts-db/src/accounts_hash.rs @@ -1148,6 +1148,51 @@ pub struct IncrementalAccountsHash(pub Hash); #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub struct AccountsDeltaHash(pub Hash); +/// Snapshot serde-safe accounts delta hash +#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, AbiExample)] +pub struct SerdeAccountsDeltaHash(pub Hash); + +impl From for AccountsDeltaHash { + fn from(accounts_delta_hash: SerdeAccountsDeltaHash) -> Self { + Self(accounts_delta_hash.0) + } +} +impl From for SerdeAccountsDeltaHash { + fn from(accounts_delta_hash: AccountsDeltaHash) -> Self { + Self(accounts_delta_hash.0) + } +} + +/// Snapshot serde-safe accounts hash +#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, AbiExample)] +pub struct SerdeAccountsHash(pub Hash); + +impl From for AccountsHash { + fn from(accounts_hash: SerdeAccountsHash) -> Self { + Self(accounts_hash.0) + } +} +impl From for SerdeAccountsHash { + fn from(accounts_hash: AccountsHash) -> Self { + Self(accounts_hash.0) + } +} + +/// Snapshot serde-safe incremental accounts hash +#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, AbiExample)] +pub struct SerdeIncrementalAccountsHash(pub Hash); + +impl From for IncrementalAccountsHash { + fn from(incremental_accounts_hash: SerdeIncrementalAccountsHash) -> Self { + Self(incremental_accounts_hash.0) + } +} +impl From for SerdeIncrementalAccountsHash { + fn from(incremental_accounts_hash: IncrementalAccountsHash) -> Self { + Self(incremental_accounts_hash.0) + } +} + #[cfg(test)] pub mod tests { use {super::*, itertools::Itertools, std::str::FromStr, tempfile::tempdir}; diff --git a/runtime/src/accounts_index.rs b/accounts-db/src/accounts_index.rs similarity index 99% rename from runtime/src/accounts_index.rs rename to accounts-db/src/accounts_index.rs index 65ad570fbd..7b84b8078a 100644 --- a/runtime/src/accounts_index.rs +++ b/accounts-db/src/accounts_index.rs @@ -449,7 +449,7 @@ pub struct RootsTracker { /// Constructed during load from snapshots. /// Updated every time we add a new root or clean/shrink an append vec into irrelevancy. /// Range is approximately the last N slots where N is # slots per epoch. - pub(crate) alive_roots: RollingBitField, + pub alive_roots: RollingBitField, uncleaned_roots: HashSet, previous_uncleaned_roots: HashSet, } @@ -677,7 +677,7 @@ pub struct AccountsIndex + Into> { program_id_index: SecondaryIndex, spl_token_mint_index: SecondaryIndex, spl_token_owner_index: SecondaryIndex, - pub(crate) roots_tracker: RwLock, + pub roots_tracker: RwLock, ongoing_scan_roots: RwLock>, // Each scan has some latest slot `S` that is the tip of the fork the scan // is iterating over. The unique id of that slot `S` is recorded here (note we don't use @@ -1420,7 +1420,7 @@ impl + Into> AccountsIndex { /// Get an account /// The latest account that appears in `ancestors` or `roots` is returned. - pub(crate) fn get( + pub fn get( &self, pubkey: &Pubkey, ancestors: Option<&Ancestors>, @@ -1996,7 +1996,8 @@ impl + Into> AccountsIndex { self.roots_tracker.read().unwrap().uncleaned_roots.len() } - #[cfg(test)] + // These functions/fields are only usable from a dev context (i.e. tests and benches) + #[cfg(feature = "dev-context-only-utils")] // filter any rooted entries and return them along with a bool that indicates // if this account has no more entries. Note this does not update the secondary // indexes! @@ -2010,6 +2011,34 @@ impl + Into> AccountsIndex { } } +// These functions/fields are only usable from a dev context (i.e. tests and benches) +#[cfg(feature = "dev-context-only-utils")] +impl AccountIndexGetResult { + pub fn unwrap(self) -> (ReadAccountMapEntry, usize) { + match self { + AccountIndexGetResult::Found(lock, size) => (lock, size), + _ => { + panic!("trying to unwrap AccountIndexGetResult with non-Success result"); + } + } + } + + pub fn is_none(&self) -> bool { + !self.is_some() + } + + pub fn is_some(&self) -> bool { + matches!(self, AccountIndexGetResult::Found(_lock, _size)) + } + + pub fn map, usize)) -> V>(self, f: F) -> Option { + match self { + AccountIndexGetResult::Found(lock, size) => Some(f((lock, size))), + _ => None, + } + } +} + #[cfg(test)] pub mod tests { use { @@ -2045,32 +2074,6 @@ pub mod tests { } } - impl AccountIndexGetResult { - pub fn unwrap(self) -> (ReadAccountMapEntry, usize) { - match self { - AccountIndexGetResult::Found(lock, size) => (lock, size), - _ => { - panic!("trying to unwrap AccountIndexGetResult with non-Success result"); - } - } - } - - pub fn is_none(&self) -> bool { - !self.is_some() - } - - pub fn is_some(&self) -> bool { - matches!(self, AccountIndexGetResult::Found(_lock, _size)) - } - - pub fn map, usize)) -> V>(self, f: F) -> Option { - match self { - AccountIndexGetResult::Found(lock, size) => Some(f((lock, size))), - _ => None, - } - } - } - fn create_dashmap_secondary_index_state() -> (usize, usize, AccountSecondaryIndexes) { { // Check that we're actually testing the correct variant diff --git a/runtime/src/accounts_index_storage.rs b/accounts-db/src/accounts_index_storage.rs similarity index 100% rename from runtime/src/accounts_index_storage.rs rename to accounts-db/src/accounts_index_storage.rs diff --git a/runtime/src/accounts_partition.rs b/accounts-db/src/accounts_partition.rs similarity index 97% rename from runtime/src/accounts_partition.rs rename to accounts-db/src/accounts_partition.rs index d97edcb058..05d3993adc 100644 --- a/runtime/src/accounts_partition.rs +++ b/accounts-db/src/accounts_partition.rs @@ -14,9 +14,9 @@ use { // Eager rent collection repeats in cyclic manner. // Each cycle is composed of number of tiny pubkey subranges // to scan, which is always multiple of the number of slots in epoch. -pub(crate) type PartitionIndex = u64; +pub type PartitionIndex = u64; type PartitionsPerCycle = u64; -pub(crate) type Partition = (PartitionIndex, PartitionIndex, PartitionsPerCycle); +pub type Partition = (PartitionIndex, PartitionIndex, PartitionsPerCycle); type RentCollectionCycleParams = ( Epoch, SlotCount, @@ -43,7 +43,7 @@ fn partition_index_from_slot_index( slot_index_in_epoch + epoch_index_in_cycle * slot_count_per_epoch } -pub(crate) fn get_partition_from_slot_indexes( +pub fn get_partition_from_slot_indexes( cycle_params: RentCollectionCycleParams, start_slot_index: SlotIndex, end_slot_index: SlotIndex, @@ -100,8 +100,9 @@ pub(crate) fn get_partition_from_slot_indexes( /// used only by filler accounts in debug path /// previous means slot - 1, not parent -#[cfg(test)] -pub(crate) fn variable_cycle_partition_from_previous_slot( +// These functions/fields are only usable from a dev context (i.e. tests and benches) +#[cfg(feature = "dev-context-only-utils")] +pub fn variable_cycle_partition_from_previous_slot( epoch_schedule: &EpochSchedule, slot: Slot, ) -> Partition { @@ -137,7 +138,7 @@ pub(crate) fn variable_cycle_partition_from_previous_slot( /// 1. 'pubkey_range_from_partition' /// 2. 'partition_from_pubkey' /// 3. this function -pub(crate) fn get_partition_end_indexes(partition: &Partition) -> Vec { +pub fn get_partition_end_indexes(partition: &Partition) -> Vec { if partition.0 == partition.1 && partition.0 == 0 { // special case for start=end=0. ie. (0, 0, N). This returns [0] vec![0] @@ -149,7 +150,7 @@ pub(crate) fn get_partition_end_indexes(partition: &Partition) -> Vec RentCollectionCycleParams { @@ -163,7 +164,7 @@ pub(crate) fn rent_single_epoch_collection_cycle_params( ) } -pub(crate) fn rent_multi_epoch_collection_cycle_params( +pub fn rent_multi_epoch_collection_cycle_params( epoch: Epoch, slot_count_per_epoch: SlotCount, first_normal_epoch: Epoch, @@ -180,7 +181,7 @@ pub(crate) fn rent_multi_epoch_collection_cycle_params( ) } -pub(crate) fn get_partitions( +pub fn get_partitions( slot: Slot, parent_slot: Slot, slot_count_in_two_day: SlotCount, @@ -221,7 +222,7 @@ pub(crate) fn get_partitions( // start_index..=end_index. But it has some exceptional cases, including // this important and valid one: // 0..=0: the first partition in the new epoch when crossing epochs -pub(crate) fn pubkey_range_from_partition( +pub fn pubkey_range_from_partition( (start_index, end_index, partition_count): Partition, ) -> RangeInclusive { assert!(start_index <= end_index); @@ -336,14 +337,14 @@ pub(crate) fn pubkey_range_from_partition( start_pubkey_final..=end_pubkey_final } -pub(crate) fn prefix_from_pubkey(pubkey: &Pubkey) -> u64 { +pub fn prefix_from_pubkey(pubkey: &Pubkey) -> u64 { const PREFIX_SIZE: usize = mem::size_of::(); u64::from_be_bytes(pubkey.as_ref()[0..PREFIX_SIZE].try_into().unwrap()) } /// This is the inverse of pubkey_range_from_partition. /// return the lowest end_index which would contain this pubkey -pub(crate) fn partition_from_pubkey( +pub fn partition_from_pubkey( pubkey: &Pubkey, partition_count: PartitionsPerCycle, ) -> PartitionIndex { diff --git a/runtime/src/accounts_update_notifier_interface.rs b/accounts-db/src/accounts_update_notifier_interface.rs similarity index 100% rename from runtime/src/accounts_update_notifier_interface.rs rename to accounts-db/src/accounts_update_notifier_interface.rs diff --git a/runtime/src/active_stats.rs b/accounts-db/src/active_stats.rs similarity index 100% rename from runtime/src/active_stats.rs rename to accounts-db/src/active_stats.rs diff --git a/runtime/src/ancestors.rs b/accounts-db/src/ancestors.rs similarity index 88% rename from runtime/src/ancestors.rs rename to accounts-db/src/ancestors.rs index 9712f1fdbb..7072640dbf 100644 --- a/runtime/src/ancestors.rs +++ b/accounts-db/src/ancestors.rs @@ -89,6 +89,36 @@ impl Ancestors { self.ancestors.max_exclusive().saturating_sub(1) } } + +// These functions/fields are only usable from a dev context (i.e. tests and benches) +#[cfg(feature = "dev-context-only-utils")] +impl std::iter::FromIterator<(Slot, usize)> for Ancestors { + fn from_iter(iter: I) -> Self + where + I: IntoIterator, + { + let mut data = Vec::new(); + for i in iter { + data.push(i); + } + Ancestors::from(data) + } +} + +#[cfg(feature = "dev-context-only-utils")] +impl From> for Ancestors { + fn from(source: Vec<(Slot, usize)>) -> Ancestors { + Ancestors::from(source.into_iter().map(|(slot, _)| slot).collect::>()) + } +} + +#[cfg(feature = "dev-context-only-utils")] +impl Ancestors { + pub fn insert(&mut self, slot: Slot, _size: usize) { + self.ancestors.insert(slot); + } +} + #[cfg(test)] pub mod tests { use { @@ -96,30 +126,6 @@ pub mod tests { std::collections::HashSet, }; - impl std::iter::FromIterator<(Slot, usize)> for Ancestors { - fn from_iter(iter: I) -> Self - where - I: IntoIterator, - { - let mut data = Vec::new(); - for i in iter { - data.push(i); - } - Ancestors::from(data) - } - } - - impl From> for Ancestors { - fn from(source: Vec<(Slot, usize)>) -> Ancestors { - Ancestors::from(source.into_iter().map(|(slot, _)| slot).collect::>()) - } - } - impl Ancestors { - pub fn insert(&mut self, slot: Slot, _size: usize) { - self.ancestors.insert(slot); - } - } - #[test] fn test_ancestors_permutations() { solana_logger::setup(); diff --git a/runtime/src/ancient_append_vecs.rs b/accounts-db/src/ancient_append_vecs.rs similarity index 100% rename from runtime/src/ancient_append_vecs.rs rename to accounts-db/src/ancient_append_vecs.rs diff --git a/runtime/src/append_vec.rs b/accounts-db/src/append_vec.rs similarity index 100% rename from runtime/src/append_vec.rs rename to accounts-db/src/append_vec.rs diff --git a/runtime/src/append_vec/test_utils.rs b/accounts-db/src/append_vec/test_utils.rs similarity index 100% rename from runtime/src/append_vec/test_utils.rs rename to accounts-db/src/append_vec/test_utils.rs diff --git a/runtime/src/blockhash_queue.rs b/accounts-db/src/blockhash_queue.rs similarity index 98% rename from runtime/src/blockhash_queue.rs rename to accounts-db/src/blockhash_queue.rs index ae24183470..a67cce0ba6 100644 --- a/runtime/src/blockhash_queue.rs +++ b/accounts-db/src/blockhash_queue.rs @@ -16,7 +16,7 @@ struct HashAge { } /// Low memory overhead, so can be cloned for every checkpoint -#[frozen_abi(digest = "J66ssCYGtWdQu5oyJxFKFeZY86nUjThBdBeXQYuRPDvE")] +#[frozen_abi(digest = "8upYCMG37Awf4FGQ5kKtZARHP1QfD2GMpQCPnwCCsxhu")] #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, AbiExample)] pub struct BlockhashQueue { /// index of last hash to be registered @@ -124,7 +124,7 @@ impl BlockhashQueue { }) } - pub(crate) fn get_max_age(&self) -> usize { + pub fn get_max_age(&self) -> usize { self.max_age } } diff --git a/runtime/src/bucket_map_holder.rs b/accounts-db/src/bucket_map_holder.rs similarity index 99% rename from runtime/src/bucket_map_holder.rs rename to accounts-db/src/bucket_map_holder.rs index aca0b0e57a..dfc77671ed 100644 --- a/runtime/src/bucket_map_holder.rs +++ b/accounts-db/src/bucket_map_holder.rs @@ -133,7 +133,7 @@ impl + Into> BucketMapHolder } /// return when the bg threads have reached an 'idle' state - pub(crate) fn wait_for_idle(&self) { + pub fn wait_for_idle(&self) { assert!(self.get_startup()); if self.disk.is_none() { return; diff --git a/runtime/src/bucket_map_holder_stats.rs b/accounts-db/src/bucket_map_holder_stats.rs similarity index 100% rename from runtime/src/bucket_map_holder_stats.rs rename to accounts-db/src/bucket_map_holder_stats.rs diff --git a/runtime/src/cache_hash_data.rs b/accounts-db/src/cache_hash_data.rs similarity index 98% rename from runtime/src/cache_hash_data.rs rename to accounts-db/src/cache_hash_data.rs index 9a0a742f2a..b0e869f760 100644 --- a/runtime/src/cache_hash_data.rs +++ b/accounts-db/src/cache_hash_data.rs @@ -23,7 +23,7 @@ pub struct Header { count: usize, } -pub(crate) struct CacheHashDataFile { +pub struct CacheHashDataFile { cell_size: u64, mmap: MmapMut, capacity: u64, @@ -31,13 +31,13 @@ pub(crate) struct CacheHashDataFile { impl CacheHashDataFile { /// return a slice of a reference to all the cache hash data from the mmapped file - pub(crate) fn get_cache_hash_data(&self) -> &[EntryType] { + pub fn get_cache_hash_data(&self) -> &[EntryType] { self.get_slice(0) } #[cfg(test)] /// Populate 'accumulator' from entire contents of the cache file. - pub(crate) fn load_all( + pub fn load_all( &self, accumulator: &mut SavedType, start_bin_index: usize, @@ -196,7 +196,7 @@ impl CacheHashData { #[cfg(test)] /// load from 'file_name' into 'accumulator' - pub(crate) fn load( + pub fn load( &self, file_name: impl AsRef, accumulator: &mut SavedType, @@ -213,7 +213,7 @@ impl CacheHashData { } /// map 'file_name' into memory - pub(crate) fn load_map( + pub fn load_map( &self, file_name: impl AsRef, ) -> Result { diff --git a/runtime/src/cache_hash_data_stats.rs b/accounts-db/src/cache_hash_data_stats.rs similarity index 100% rename from runtime/src/cache_hash_data_stats.rs rename to accounts-db/src/cache_hash_data_stats.rs diff --git a/runtime/src/contains.rs b/accounts-db/src/contains.rs similarity index 100% rename from runtime/src/contains.rs rename to accounts-db/src/contains.rs diff --git a/runtime/src/epoch_accounts_hash.rs b/accounts-db/src/epoch_accounts_hash.rs similarity index 100% rename from runtime/src/epoch_accounts_hash.rs rename to accounts-db/src/epoch_accounts_hash.rs diff --git a/runtime/src/epoch_accounts_hash/manager.rs b/accounts-db/src/epoch_accounts_hash/manager.rs similarity index 100% rename from runtime/src/epoch_accounts_hash/manager.rs rename to accounts-db/src/epoch_accounts_hash/manager.rs diff --git a/runtime/src/hardened_unpack.rs b/accounts-db/src/hardened_unpack.rs similarity index 100% rename from runtime/src/hardened_unpack.rs rename to accounts-db/src/hardened_unpack.rs diff --git a/runtime/src/in_mem_accounts_index.rs b/accounts-db/src/in_mem_accounts_index.rs similarity index 99% rename from runtime/src/in_mem_accounts_index.rs rename to accounts-db/src/in_mem_accounts_index.rs index 652fb0f854..cfc8c697e9 100644 --- a/runtime/src/in_mem_accounts_index.rs +++ b/accounts-db/src/in_mem_accounts_index.rs @@ -94,7 +94,7 @@ pub struct InMemAccountsIndex + Into< bucket: Option>>, // pubkey ranges that this bin must hold in the cache while the range is present in this vec - pub(crate) cache_ranges_held: CacheRangesHeld, + pub cache_ranges_held: CacheRangesHeld, // incremented each time stop_evictions is changed stop_evictions_changes: AtomicU64, // true while ranges are being manipulated. Used to keep an async flush from removing things while a range is being held. @@ -322,7 +322,7 @@ impl + Into> InMemAccountsIndex( + pub fn get_internal( &self, pubkey: &K, // return true if item should be added to in_mem cache @@ -545,7 +545,7 @@ impl + Into> InMemAccountsIndex, new_value: (Slot, T), other_slot: Option, @@ -920,7 +920,7 @@ impl + Into> InMemAccountsIndex + Into> InMemAccountsIndex 1 /// These were collected for this bin when we did batch inserts in the bg flush threads. /// Insert these into the in-mem index, then return the duplicate (Slot, Pubkey) - pub(crate) fn populate_and_retrieve_duplicate_keys_from_startup(&self) -> Vec<(Slot, Pubkey)> { + pub fn populate_and_retrieve_duplicate_keys_from_startup(&self) -> Vec<(Slot, Pubkey)> { // in order to return accurate and complete duplicates, we must have nothing left remaining to insert assert!(self.startup_info.insert.lock().unwrap().is_empty()); diff --git a/runtime/src/inline_spl_token.rs b/accounts-db/src/inline_spl_token.rs similarity index 97% rename from runtime/src/inline_spl_token.rs rename to accounts-db/src/inline_spl_token.rs index 721cdab8b7..0e936c5f40 100644 --- a/runtime/src/inline_spl_token.rs +++ b/accounts-db/src/inline_spl_token.rs @@ -3,7 +3,7 @@ use solana_sdk::pubkey::{Pubkey, PUBKEY_BYTES}; solana_sdk::declare_id!("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA"); -pub(crate) mod program_v3_4_0 { +pub mod program_v3_4_0 { solana_sdk::declare_id!("NToK4t5AQzxPNpUA84DkxgfXaVDbDQQjpHKCqsbY46B"); } @@ -23,7 +23,7 @@ pub const SPL_TOKEN_ACCOUNT_MINT_OFFSET: usize = 0; pub const SPL_TOKEN_ACCOUNT_OWNER_OFFSET: usize = 32; const SPL_TOKEN_ACCOUNT_LENGTH: usize = 165; -pub(crate) trait GenericTokenAccount { +pub trait GenericTokenAccount { fn valid_account_data(account_data: &[u8]) -> bool; // Call after account length has already been verified diff --git a/runtime/src/inline_spl_token_2022.rs b/accounts-db/src/inline_spl_token_2022.rs similarity index 100% rename from runtime/src/inline_spl_token_2022.rs rename to accounts-db/src/inline_spl_token_2022.rs diff --git a/accounts-db/src/lib.rs b/accounts-db/src/lib.rs new file mode 100644 index 0000000000..8f62ea8ffa --- /dev/null +++ b/accounts-db/src/lib.rs @@ -0,0 +1,59 @@ +#![cfg_attr(RUSTC_WITH_SPECIALIZATION, feature(min_specialization))] +#![allow(clippy::integer_arithmetic)] + +#[macro_use] +extern crate lazy_static; + +pub mod account_info; +pub mod account_overrides; +pub mod account_rent_state; +pub mod account_storage; +pub mod accounts; +pub mod accounts_cache; +pub mod accounts_db; +pub mod accounts_file; +pub mod accounts_hash; +pub mod accounts_index; +pub mod accounts_index_storage; +pub mod accounts_partition; +pub mod accounts_update_notifier_interface; +pub mod active_stats; +pub mod ancestors; +pub mod ancient_append_vecs; +pub mod append_vec; +pub mod blockhash_queue; +pub mod bucket_map_holder; +pub mod bucket_map_holder_stats; +pub mod cache_hash_data; +pub mod cache_hash_data_stats; +pub mod contains; +pub mod epoch_accounts_hash; +pub mod hardened_unpack; +pub mod in_mem_accounts_index; +pub mod inline_spl_token; +pub mod inline_spl_token_2022; +pub mod nonce_info; +pub mod partitioned_rewards; +mod pubkey_bins; +mod read_only_accounts_cache; +pub mod rent_collector; +pub mod rent_debits; +mod rolling_bit_field; +pub mod secondary_index; +pub mod shared_buffer_reader; +pub mod sorted_storages; +pub mod stake_rewards; +pub mod storable_accounts; +pub mod tiered_storage; +pub mod transaction_error_metrics; +pub mod transaction_results; +mod verify_accounts_hash_in_background; +pub mod waitable_condvar; + +#[macro_use] +extern crate solana_metrics; +#[macro_use] +extern crate serde_derive; + +#[macro_use] +extern crate solana_frozen_abi_macro; diff --git a/runtime/src/nonce_info.rs b/accounts-db/src/nonce_info.rs similarity index 100% rename from runtime/src/nonce_info.rs rename to accounts-db/src/nonce_info.rs diff --git a/runtime/src/partitioned_rewards.rs b/accounts-db/src/partitioned_rewards.rs similarity index 93% rename from runtime/src/partitioned_rewards.rs rename to accounts-db/src/partitioned_rewards.rs index f9c286d01c..c638b047ca 100644 --- a/runtime/src/partitioned_rewards.rs +++ b/accounts-db/src/partitioned_rewards.rs @@ -6,22 +6,22 @@ use solana_sdk::clock::Slot; #[derive(Debug)] /// Configuration options for partitioned epoch rewards. /// This struct allows various forms of testing, especially prior to feature activation. -pub(crate) struct PartitionedEpochRewardsConfig { +pub struct PartitionedEpochRewardsConfig { /// Number of blocks for reward calculation and storing vote accounts. /// Distributing rewards to stake accounts begins AFTER this many blocks. /// Normally, this will be 1. /// if force_one_slot_partitioned_rewards, this will be 0 (ie. we take 0 blocks just for reward calculation) - pub(crate) reward_calculation_num_blocks: Slot, + pub reward_calculation_num_blocks: Slot, /// number of stake accounts to store in one block during partitioned reward interval /// normally, this is a number tuned for reasonable performance, such as 4096 accounts/block /// if force_one_slot_partitioned_rewards, this will usually be u64::MAX so that all stake accounts are written in the first block - pub(crate) stake_account_stores_per_block: Slot, + pub stake_account_stores_per_block: Slot, /// if true, end of epoch bank rewards will force using partitioned rewards distribution. /// see `set_test_enable_partitioned_rewards` - pub(crate) test_enable_partitioned_rewards: bool, + pub test_enable_partitioned_rewards: bool, /// if true, end of epoch non-partitioned bank rewards will test the partitioned rewards distribution vote and stake accounts /// This has a significant performance impact on the first slot in each new epoch. - pub(crate) test_compare_partitioned_epoch_rewards: bool, + pub test_compare_partitioned_epoch_rewards: bool, } impl Default for PartitionedEpochRewardsConfig { @@ -55,7 +55,7 @@ pub enum TestPartitionedEpochRewards { #[allow(dead_code)] impl PartitionedEpochRewardsConfig { - pub(crate) fn new(test: TestPartitionedEpochRewards) -> Self { + pub fn new(test: TestPartitionedEpochRewards) -> Self { match test { TestPartitionedEpochRewards::None => Self::default(), TestPartitionedEpochRewards::CompareResults => { diff --git a/runtime/src/pubkey_bins.rs b/accounts-db/src/pubkey_bins.rs similarity index 100% rename from runtime/src/pubkey_bins.rs rename to accounts-db/src/pubkey_bins.rs diff --git a/runtime/src/read_only_accounts_cache.rs b/accounts-db/src/read_only_accounts_cache.rs similarity index 97% rename from runtime/src/read_only_accounts_cache.rs rename to accounts-db/src/read_only_accounts_cache.rs index d0e48480fe..bd1720b760 100644 --- a/runtime/src/read_only_accounts_cache.rs +++ b/accounts-db/src/read_only_accounts_cache.rs @@ -27,7 +27,7 @@ struct ReadOnlyAccountCacheEntry { } #[derive(Debug)] -pub(crate) struct ReadOnlyAccountsCache { +pub struct ReadOnlyAccountsCache { cache: DashMap, // When an item is first entered into the cache, it is added to the end of // the queue. Also each time an entry is looked up from the cache it is @@ -44,7 +44,7 @@ pub(crate) struct ReadOnlyAccountsCache { } impl ReadOnlyAccountsCache { - pub(crate) fn new(max_data_size: usize) -> Self { + pub fn new(max_data_size: usize) -> Self { Self { max_data_size, cache: DashMap::default(), @@ -102,7 +102,7 @@ impl ReadOnlyAccountsCache { CACHE_ENTRY_SIZE + account.data().len() } - pub(crate) fn store(&self, pubkey: Pubkey, slot: Slot, account: AccountSharedData) { + pub fn store(&self, pubkey: Pubkey, slot: Slot, account: AccountSharedData) { let key = (pubkey, slot); let account_size = self.account_size(&account); self.data_size.fetch_add(account_size, Ordering::Relaxed); @@ -138,7 +138,7 @@ impl ReadOnlyAccountsCache { self.evicts.fetch_add(num_evicts, Ordering::Relaxed); } - pub(crate) fn remove(&self, pubkey: Pubkey, slot: Slot) -> Option { + pub fn remove(&self, pubkey: Pubkey, slot: Slot) -> Option { let (_, entry) = self.cache.remove(&(pubkey, slot))?; // self.queue should be modified only after removing the entry from the // cache, so that this is still safe if another thread writes to the @@ -149,11 +149,11 @@ impl ReadOnlyAccountsCache { Some(entry.account) } - pub(crate) fn cache_len(&self) -> usize { + pub fn cache_len(&self) -> usize { self.cache.len() } - pub(crate) fn data_size(&self) -> usize { + pub fn data_size(&self) -> usize { self.data_size.load(Ordering::Relaxed) } diff --git a/runtime/src/rent_collector.rs b/accounts-db/src/rent_collector.rs similarity index 97% rename from runtime/src/rent_collector.rs rename to accounts-db/src/rent_collector.rs index ea0bea42b2..c14a2a85a3 100644 --- a/runtime/src/rent_collector.rs +++ b/accounts-db/src/rent_collector.rs @@ -49,7 +49,7 @@ enum RentResult { } impl RentCollector { - pub(crate) fn new( + pub fn new( epoch: Epoch, epoch_schedule: EpochSchedule, slots_per_year: f64, @@ -63,7 +63,7 @@ impl RentCollector { } } - pub(crate) fn clone_with_epoch(&self, epoch: Epoch) -> Self { + pub fn clone_with_epoch(&self, epoch: Epoch) -> Self { Self { epoch, ..self.clone() @@ -71,18 +71,14 @@ impl RentCollector { } /// true if it is easy to determine this account should consider having rent collected from it - pub(crate) fn should_collect_rent( - &self, - address: &Pubkey, - account: &impl ReadableAccount, - ) -> bool { + pub fn should_collect_rent(&self, address: &Pubkey, account: &impl ReadableAccount) -> bool { !(account.executable() // executable accounts must be rent-exempt balance || *address == incinerator::id()) } /// given an account that 'should_collect_rent' /// returns (amount rent due, is_exempt_from_rent) - pub(crate) fn get_rent_due(&self, account: &impl ReadableAccount) -> RentDue { + pub fn get_rent_due(&self, account: &impl ReadableAccount) -> RentDue { if self .rent .is_exempt(account.lamports(), account.data().len()) @@ -111,7 +107,7 @@ impl RentCollector { // This is NOT thread safe at some level. If we try to collect from the same account in // parallel, we may collect twice. #[must_use = "add to Bank::collected_rent"] - pub(crate) fn collect_from_existing_account( + pub fn collect_from_existing_account( &self, address: &Pubkey, account: &mut AccountSharedData, @@ -188,11 +184,11 @@ impl RentCollector { /// Information computed during rent collection #[derive(Debug, Default, Copy, Clone, Eq, PartialEq)] -pub(crate) struct CollectedInfo { +pub struct CollectedInfo { /// Amount of rent collected from account - pub(crate) rent_amount: u64, + pub rent_amount: u64, /// Size of data reclaimed from account (happens when account's lamports go to zero) - pub(crate) account_data_len_reclaimed: u64, + pub account_data_len_reclaimed: u64, } impl std::ops::Add for CollectedInfo { diff --git a/runtime/src/rent_debits.rs b/accounts-db/src/rent_debits.rs similarity index 77% rename from runtime/src/rent_debits.rs rename to accounts-db/src/rent_debits.rs index 479f4a1629..75d8eddec1 100644 --- a/runtime/src/rent_debits.rs +++ b/accounts-db/src/rent_debits.rs @@ -1,11 +1,11 @@ use { - crate::bank::RewardInfo, + crate::stake_rewards::RewardInfo, solana_sdk::{pubkey::Pubkey, reward_type::RewardType}, std::collections::HashMap, }; #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct RentDebit { +pub struct RentDebit { rent_collected: u64, post_balance: u64, } @@ -27,18 +27,24 @@ impl RentDebit { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct RentDebits(HashMap); impl RentDebits { - pub(crate) fn get_account_rent_debit(&self, address: &Pubkey) -> u64 { + pub fn get_account_rent_debit(&self, address: &Pubkey) -> u64 { self.0 .get(address) .map(|r| r.rent_collected) .unwrap_or_default() } - #[cfg(test)] - pub(crate) fn len(&self) -> usize { + // These functions/fields are only usable from a dev context (i.e. tests and benches) + #[cfg(feature = "dev-context-only-utils")] + pub fn len(&self) -> usize { self.0.len() } + #[cfg(feature = "dev-context-only-utils")] + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + pub fn insert(&mut self, address: &Pubkey, rent_collected: u64, post_balance: u64) { if rent_collected != 0 { self.0.insert( diff --git a/runtime/src/rolling_bit_field.rs b/accounts-db/src/rolling_bit_field.rs similarity index 100% rename from runtime/src/rolling_bit_field.rs rename to accounts-db/src/rolling_bit_field.rs diff --git a/runtime/src/secondary_index.rs b/accounts-db/src/secondary_index.rs similarity index 100% rename from runtime/src/secondary_index.rs rename to accounts-db/src/secondary_index.rs diff --git a/runtime/src/shared_buffer_reader.rs b/accounts-db/src/shared_buffer_reader.rs similarity index 100% rename from runtime/src/shared_buffer_reader.rs rename to accounts-db/src/shared_buffer_reader.rs diff --git a/runtime/src/sorted_storages.rs b/accounts-db/src/sorted_storages.rs similarity index 100% rename from runtime/src/sorted_storages.rs rename to accounts-db/src/sorted_storages.rs diff --git a/accounts-db/src/stake_rewards.rs b/accounts-db/src/stake_rewards.rs new file mode 100644 index 0000000000..99d1dca824 --- /dev/null +++ b/accounts-db/src/stake_rewards.rs @@ -0,0 +1,115 @@ +//! Code for stake and vote rewards + +use { + crate::{accounts_db::IncludeSlotInHash, storable_accounts::StorableAccounts}, + solana_sdk::{ + account::AccountSharedData, clock::Slot, pubkey::Pubkey, reward_type::RewardType, + }, +}; + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, AbiExample, Clone, Copy)] +pub struct RewardInfo { + pub reward_type: RewardType, + /// Reward amount + pub lamports: i64, + /// Account balance in lamports after `lamports` was applied + pub post_balance: u64, + /// Vote account commission when the reward was credited, only present for voting and staking rewards + pub commission: Option, +} + +#[derive(AbiExample, Debug, Serialize, Deserialize, Clone, PartialEq)] +pub struct StakeReward { + pub stake_pubkey: Pubkey, + pub stake_reward_info: RewardInfo, + pub stake_account: AccountSharedData, +} + +impl StakeReward { + pub fn get_stake_reward(&self) -> i64 { + self.stake_reward_info.lamports + } +} + +/// allow [StakeReward] to be passed to `StoreAccounts` directly without copies or vec construction +impl<'a> StorableAccounts<'a, AccountSharedData> for (Slot, &'a [StakeReward], IncludeSlotInHash) { + fn pubkey(&self, index: usize) -> &Pubkey { + &self.1[index].stake_pubkey + } + fn account(&self, index: usize) -> &AccountSharedData { + &self.1[index].stake_account + } + fn slot(&self, _index: usize) -> Slot { + // per-index slot is not unique per slot when per-account slot is not included in the source data + self.target_slot() + } + fn target_slot(&self) -> Slot { + self.0 + } + fn len(&self) -> usize { + self.1.len() + } + fn include_slot_in_hash(&self) -> IncludeSlotInHash { + self.2 + } +} + +#[cfg(feature = "dev-context-only-utils")] +use { + rand::Rng, + solana_sdk::{ + account::WritableAccount, + rent::Rent, + signature::{Keypair, Signer}, + }, + solana_stake_program::stake_state, + solana_vote_program::vote_state, +}; + +// These functions/fields are only usable from a dev context (i.e. tests and benches) +#[cfg(feature = "dev-context-only-utils")] +impl StakeReward { + pub fn new_random() -> Self { + let mut rng = rand::thread_rng(); + + let rent = Rent::free(); + + let validator_pubkey = solana_sdk::pubkey::new_rand(); + let validator_stake_lamports = 20; + let validator_staking_keypair = Keypair::new(); + let validator_voting_keypair = Keypair::new(); + + let validator_vote_account = vote_state::create_account( + &validator_voting_keypair.pubkey(), + &validator_pubkey, + 10, + validator_stake_lamports, + ); + + let validator_stake_account = stake_state::create_account( + &validator_staking_keypair.pubkey(), + &validator_voting_keypair.pubkey(), + &validator_vote_account, + &rent, + validator_stake_lamports, + ); + + Self { + stake_pubkey: Pubkey::new_unique(), + stake_reward_info: RewardInfo { + reward_type: RewardType::Staking, + lamports: rng.gen_range(1, 200), + post_balance: 0, /* unused atm */ + commission: None, /* unused atm */ + }, + + stake_account: validator_stake_account, + } + } + + pub fn credit(&mut self, amount: u64) { + self.stake_reward_info.lamports = amount as i64; + self.stake_reward_info.post_balance += amount; + self.stake_account.checked_add_lamports(amount).unwrap(); + } +} diff --git a/runtime/src/storable_accounts.rs b/accounts-db/src/storable_accounts.rs similarity index 99% rename from runtime/src/storable_accounts.rs rename to accounts-db/src/storable_accounts.rs index 6cc78ff1ee..f9fa74deb9 100644 --- a/runtime/src/storable_accounts.rs +++ b/accounts-db/src/storable_accounts.rs @@ -203,7 +203,7 @@ pub struct StorableAccountsBySlot<'a> { impl<'a> StorableAccountsBySlot<'a> { #[allow(dead_code)] /// each element of slots_and_accounts is (source slot, accounts moving FROM source slot) - pub(crate) fn new( + pub fn new( target_slot: Slot, slots_and_accounts: &'a [(Slot, &'a [&'a StoredAccountMeta<'a>])], include_slot_in_hash: IncludeSlotInHash, diff --git a/runtime/src/tiered_storage.rs b/accounts-db/src/tiered_storage.rs similarity index 100% rename from runtime/src/tiered_storage.rs rename to accounts-db/src/tiered_storage.rs diff --git a/runtime/src/tiered_storage/byte_block.rs b/accounts-db/src/tiered_storage/byte_block.rs similarity index 100% rename from runtime/src/tiered_storage/byte_block.rs rename to accounts-db/src/tiered_storage/byte_block.rs diff --git a/runtime/src/tiered_storage/error.rs b/accounts-db/src/tiered_storage/error.rs similarity index 100% rename from runtime/src/tiered_storage/error.rs rename to accounts-db/src/tiered_storage/error.rs diff --git a/runtime/src/tiered_storage/file.rs b/accounts-db/src/tiered_storage/file.rs similarity index 100% rename from runtime/src/tiered_storage/file.rs rename to accounts-db/src/tiered_storage/file.rs diff --git a/runtime/src/tiered_storage/footer.rs b/accounts-db/src/tiered_storage/footer.rs similarity index 100% rename from runtime/src/tiered_storage/footer.rs rename to accounts-db/src/tiered_storage/footer.rs diff --git a/runtime/src/tiered_storage/hot.rs b/accounts-db/src/tiered_storage/hot.rs similarity index 100% rename from runtime/src/tiered_storage/hot.rs rename to accounts-db/src/tiered_storage/hot.rs diff --git a/runtime/src/tiered_storage/index.rs b/accounts-db/src/tiered_storage/index.rs similarity index 100% rename from runtime/src/tiered_storage/index.rs rename to accounts-db/src/tiered_storage/index.rs diff --git a/runtime/src/tiered_storage/meta.rs b/accounts-db/src/tiered_storage/meta.rs similarity index 100% rename from runtime/src/tiered_storage/meta.rs rename to accounts-db/src/tiered_storage/meta.rs diff --git a/runtime/src/tiered_storage/mmap_utils.rs b/accounts-db/src/tiered_storage/mmap_utils.rs similarity index 100% rename from runtime/src/tiered_storage/mmap_utils.rs rename to accounts-db/src/tiered_storage/mmap_utils.rs diff --git a/runtime/src/tiered_storage/readable.rs b/accounts-db/src/tiered_storage/readable.rs similarity index 94% rename from runtime/src/tiered_storage/readable.rs rename to accounts-db/src/tiered_storage/readable.rs index 5981ae30e4..686f622ea0 100644 --- a/runtime/src/tiered_storage/readable.rs +++ b/accounts-db/src/tiered_storage/readable.rs @@ -16,16 +16,16 @@ use { #[derive(PartialEq, Eq, Debug)] pub struct TieredReadableAccount<'accounts_file, M: TieredAccountMeta> { /// TieredAccountMeta - pub(crate) meta: &'accounts_file M, + pub meta: &'accounts_file M, /// The address of the account - pub(crate) address: &'accounts_file Pubkey, + pub address: &'accounts_file Pubkey, /// The address of the account owner - pub(crate) owner: &'accounts_file Pubkey, + pub owner: &'accounts_file Pubkey, /// The index for accessing the account inside its belonging AccountsFile - pub(crate) index: usize, + pub index: usize, /// The account block that contains this account. Note that this account /// block may be shared with other accounts. - pub(crate) account_block: &'accounts_file [u8], + pub account_block: &'accounts_file [u8], } impl<'accounts_file, M: TieredAccountMeta> TieredReadableAccount<'accounts_file, M> { diff --git a/runtime/src/tiered_storage/writer.rs b/accounts-db/src/tiered_storage/writer.rs similarity index 100% rename from runtime/src/tiered_storage/writer.rs rename to accounts-db/src/tiered_storage/writer.rs diff --git a/runtime/src/transaction_error_metrics.rs b/accounts-db/src/transaction_error_metrics.rs similarity index 100% rename from runtime/src/transaction_error_metrics.rs rename to accounts-db/src/transaction_error_metrics.rs diff --git a/runtime/src/transaction_results.rs b/accounts-db/src/transaction_results.rs similarity index 100% rename from runtime/src/transaction_results.rs rename to accounts-db/src/transaction_results.rs diff --git a/runtime/src/verify_accounts_hash_in_background.rs b/accounts-db/src/verify_accounts_hash_in_background.rs similarity index 93% rename from runtime/src/verify_accounts_hash_in_background.rs rename to accounts-db/src/verify_accounts_hash_in_background.rs index 121d884f3e..d4676cfe12 100644 --- a/runtime/src/verify_accounts_hash_in_background.rs +++ b/accounts-db/src/verify_accounts_hash_in_background.rs @@ -12,9 +12,9 @@ use { }; #[derive(Debug)] -pub(crate) struct VerifyAccountsHashInBackground { +pub struct VerifyAccountsHashInBackground { /// true when verification has completed or never had to run in background - pub(crate) verified: Arc, + pub verified: Arc, /// enable waiting for verification to become complete complete: Arc, /// thread doing verification @@ -39,14 +39,14 @@ impl Default for VerifyAccountsHashInBackground { impl VerifyAccountsHashInBackground { /// start the bg thread to do the verification - pub(crate) fn start(&self, start: impl FnOnce() -> JoinHandle) { + pub fn start(&self, start: impl FnOnce() -> JoinHandle) { // note that we're not verified before self.verified.store(false, Ordering::Release); *self.thread.lock().unwrap() = Some(start()); } /// notify that the bg process has completed - pub(crate) fn background_finished(&self) { + pub fn background_finished(&self) { self.complete.notify_all(); self.background_completed.store(true, Ordering::Release); } @@ -54,7 +54,7 @@ impl VerifyAccountsHashInBackground { /// notify that verification was completed successfully /// This can occur because it completed in the background /// or if the verification was run in the foreground. - pub(crate) fn verification_complete(&self) { + pub fn verification_complete(&self) { self.verified.store(true, Ordering::Release); } @@ -76,7 +76,7 @@ impl VerifyAccountsHashInBackground { /// return true if bg hash verification is complete /// return false if bg hash verification has not completed yet /// if hash verification failed, a panic will occur - pub(crate) fn check_complete(&self) -> bool { + pub fn check_complete(&self) -> bool { if self.verified.load(Ordering::Acquire) { // already completed return true; @@ -95,7 +95,7 @@ impl VerifyAccountsHashInBackground { } #[cfg(test)] -pub(crate) mod tests { +pub mod tests { use {super::*, std::thread::Builder}; #[test] diff --git a/runtime/src/waitable_condvar.rs b/accounts-db/src/waitable_condvar.rs similarity index 100% rename from runtime/src/waitable_condvar.rs rename to accounts-db/src/waitable_condvar.rs diff --git a/banks-server/Cargo.toml b/banks-server/Cargo.toml index 4c29dcd30d..1404d88b5c 100644 --- a/banks-server/Cargo.toml +++ b/banks-server/Cargo.toml @@ -13,6 +13,7 @@ edition = { workspace = true } bincode = { workspace = true } crossbeam-channel = { workspace = true } futures = { workspace = true } +solana-accounts-db = { workspace = true } solana-banks-interface = { workspace = true } solana-client = { workspace = true } solana-runtime = { workspace = true } diff --git a/banks-server/src/banks_server.rs b/banks-server/src/banks_server.rs index c491663105..53dc3b8389 100644 --- a/banks-server/src/banks_server.rs +++ b/banks-server/src/banks_server.rs @@ -2,6 +2,7 @@ use { bincode::{deserialize, serialize}, crossbeam_channel::{unbounded, Receiver, Sender}, futures::{future, prelude::stream::StreamExt}, + solana_accounts_db::transaction_results::TransactionExecutionResult, solana_banks_interface::{ Banks, BanksRequest, BanksResponse, BanksTransactionResultWithMetadata, BanksTransactionResultWithSimulation, TransactionConfirmationStatus, TransactionMetadata, @@ -12,7 +13,6 @@ use { bank::{Bank, TransactionSimulationResult}, bank_forks::BankForks, commitment::BlockCommitmentCache, - transaction_results::TransactionExecutionResult, }, solana_sdk::{ account::Account, diff --git a/core/Cargo.toml b/core/Cargo.toml index 1e93de93b6..65df2bcaf9 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -37,6 +37,7 @@ rayon = { workspace = true } rolling-file = { workspace = true } serde = { workspace = true } serde_derive = { workspace = true } +solana-accounts-db = { workspace = true } solana-address-lookup-table-program = { workspace = true } solana-bloom = { workspace = true } solana-client = { workspace = true } diff --git a/core/src/accounts_hash_verifier.rs b/core/src/accounts_hash_verifier.rs index 7bf48047ea..b09cd90018 100644 --- a/core/src/accounts_hash_verifier.rs +++ b/core/src/accounts_hash_verifier.rs @@ -5,14 +5,17 @@ use { crossbeam_channel::{Receiver, Sender}, - solana_gossip::cluster_info::{ClusterInfo, MAX_ACCOUNTS_HASHES}, - solana_measure::measure_us, - solana_runtime::{ + solana_accounts_db::{ accounts_db::CalcAccountsHashFlavor, accounts_hash::{ AccountsHash, AccountsHashEnum, CalcAccountsHashConfig, HashStats, IncrementalAccountsHash, }, + sorted_storages::SortedStorages, + }, + solana_gossip::cluster_info::{ClusterInfo, MAX_ACCOUNTS_HASHES}, + solana_measure::measure_us, + solana_runtime::{ serde_snapshot::BankIncrementalSnapshotPersistence, snapshot_config::SnapshotConfig, snapshot_package::{ @@ -20,7 +23,6 @@ use { SnapshotType, }, snapshot_utils, - sorted_storages::SortedStorages, }, solana_sdk::{ clock::{Slot, DEFAULT_MS_PER_SLOT}, diff --git a/core/src/banking_stage/committer.rs b/core/src/banking_stage/committer.rs index 1f82be37d0..26f5e56054 100644 --- a/core/src/banking_stage/committer.rs +++ b/core/src/banking_stage/committer.rs @@ -1,17 +1,19 @@ use { super::leader_slot_timing_metrics::LeaderExecuteAndCommitTimings, itertools::Itertools, + solana_accounts_db::{ + accounts::TransactionLoadResult, + transaction_results::{TransactionExecutionResult, TransactionResults}, + }, solana_ledger::{ blockstore_processor::TransactionStatusSender, token_balances::collect_token_balances, }, solana_measure::measure_us, solana_runtime::{ - accounts::TransactionLoadResult, bank::{Bank, CommitTransactionCounts, TransactionBalancesSet}, bank_utils, prioritization_fee_cache::PrioritizationFeeCache, transaction_batch::TransactionBatch, - transaction_results::{TransactionExecutionResult, TransactionResults}, vote_sender_types::ReplayVoteSender, }, solana_sdk::{pubkey::Pubkey, saturating_add_assign}, diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index bb5b135cd9..12f4b71023 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -9,6 +9,10 @@ use { BankingStageStats, }, itertools::Itertools, + solana_accounts_db::{ + transaction_error_metrics::TransactionErrorMetrics, + transaction_results::TransactionCheckResult, + }, solana_ledger::token_balances::collect_token_balances, solana_measure::{measure::Measure, measure_us}, solana_poh::poh_recorder::{ @@ -19,8 +23,6 @@ use { solana_runtime::{ bank::{Bank, LoadAndExecuteTransactionsOutput}, transaction_batch::TransactionBatch, - transaction_error_metrics::TransactionErrorMetrics, - transaction_results::TransactionCheckResult, }, solana_sdk::{ clock::{Slot, FORWARD_TRANSACTIONS_TO_LEADER_AT_SLOT_OFFSET, MAX_PROCESSING_AGE}, diff --git a/core/src/banking_stage/leader_slot_metrics.rs b/core/src/banking_stage/leader_slot_metrics.rs index b667e64e36..7620776664 100644 --- a/core/src/banking_stage/leader_slot_metrics.rs +++ b/core/src/banking_stage/leader_slot_metrics.rs @@ -3,8 +3,8 @@ use { leader_slot_timing_metrics::{LeaderExecuteAndCommitTimings, LeaderSlotTimingMetrics}, unprocessed_transaction_storage::InsertPacketBatchSummary, }, + solana_accounts_db::transaction_error_metrics::*, solana_poh::poh_recorder::BankStart, - solana_runtime::transaction_error_metrics::*, solana_sdk::{clock::Slot, saturating_add_assign}, std::time::Instant, }; diff --git a/core/src/repair/repair_weight.rs b/core/src/repair/repair_weight.rs index c0b816e5e5..6838021d75 100644 --- a/core/src/repair/repair_weight.rs +++ b/core/src/repair/repair_weight.rs @@ -978,11 +978,12 @@ mod test { use { super::*, itertools::Itertools, + solana_accounts_db::contains::Contains, solana_ledger::{ blockstore::{make_chaining_slot_entries, Blockstore}, get_tmp_ledger_path, }, - solana_runtime::{bank::Bank, bank_utils, contains::Contains}, + solana_runtime::{bank::Bank, bank_utils}, solana_sdk::hash::Hash, trees::tr, }; diff --git a/core/src/rewards_recorder_service.rs b/core/src/rewards_recorder_service.rs index b671bfe740..f78b8bab26 100644 --- a/core/src/rewards_recorder_service.rs +++ b/core/src/rewards_recorder_service.rs @@ -1,7 +1,7 @@ use { crossbeam_channel::{Receiver, RecvTimeoutError, Sender}, + solana_accounts_db::stake_rewards::RewardInfo, solana_ledger::blockstore::Blockstore, - solana_runtime::bank::RewardInfo, solana_sdk::{clock::Slot, pubkey::Pubkey}, solana_transaction_status::Reward, std::{ diff --git a/core/src/validator.rs b/core/src/validator.rs index 0e064200eb..7bc8a8f12c 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -31,6 +31,12 @@ use { crossbeam_channel::{bounded, unbounded, Receiver}, lazy_static::lazy_static, quinn::Endpoint, + solana_accounts_db::{ + accounts_db::{AccountShrinkThreshold, AccountsDbConfig}, + accounts_index::AccountSecondaryIndexes, + accounts_update_notifier_interface::AccountsUpdateNotifier, + hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, + }, solana_client::connection_cache::{ConnectionCache, Protocol}, solana_entry::poh::compute_hash_time_ns, solana_geyser_plugin_manager::{ @@ -83,13 +89,9 @@ use { AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService, DroppedSlotsReceiver, PrunedBanksRequestHandler, SnapshotRequestHandler, }, - accounts_db::{AccountShrinkThreshold, AccountsDbConfig}, - accounts_index::AccountSecondaryIndexes, - accounts_update_notifier_interface::AccountsUpdateNotifier, bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache, - hardened_unpack::{open_genesis_config, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, prioritization_fee_cache::PrioritizationFeeCache, runtime_config::RuntimeConfig, snapshot_archive_info::SnapshotArchiveInfoGetter, @@ -1917,7 +1919,7 @@ fn maybe_warp_slot( &root_bank, &Pubkey::default(), warp_slot, - solana_runtime::accounts_db::CalcAccountsHashDataSource::Storages, + solana_accounts_db::accounts_db::CalcAccountsHashDataSource::Storages, )); bank_forks.set_root( warp_slot, diff --git a/core/tests/epoch_accounts_hash.rs b/core/tests/epoch_accounts_hash.rs index 21922113b4..7d56ffa42d 100755 --- a/core/tests/epoch_accounts_hash.rs +++ b/core/tests/epoch_accounts_hash.rs @@ -1,6 +1,14 @@ use { crate::snapshot_utils::create_tmp_accounts_dir_for_tests, log::*, + solana_accounts_db::{ + accounts_db::{ + AccountShrinkThreshold, CalcAccountsHashDataSource, INCLUDE_SLOT_IN_HASH_TESTS, + }, + accounts_hash::CalcAccountsHashConfig, + accounts_index::AccountSecondaryIndexes, + epoch_accounts_hash::EpochAccountsHash, + }, solana_core::{ accounts_hash_verifier::AccountsHashVerifier, snapshot_packager_service::SnapshotPackagerService, @@ -11,14 +19,8 @@ use { AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService, DroppedSlotsReceiver, PrunedBanksRequestHandler, SnapshotRequestHandler, }, - accounts_db::{ - AccountShrinkThreshold, CalcAccountsHashDataSource, INCLUDE_SLOT_IN_HASH_TESTS, - }, - accounts_hash::CalcAccountsHashConfig, - accounts_index::AccountSecondaryIndexes, bank::{epoch_accounts_hash_utils, Bank, BankTestConfig}, bank_forks::BankForks, - epoch_accounts_hash::EpochAccountsHash, genesis_utils::{self, GenesisConfigInfo}, runtime_config::RuntimeConfig, snapshot_archive_info::SnapshotArchiveInfoGetter, diff --git a/core/tests/snapshots.rs b/core/tests/snapshots.rs index 55b35bd098..c51a8767bc 100644 --- a/core/tests/snapshots.rs +++ b/core/tests/snapshots.rs @@ -6,6 +6,12 @@ use { fs_extra::dir::CopyOptions, itertools::Itertools, log::{info, trace}, + solana_accounts_db::{ + accounts_db::{self, CalcAccountsHashDataSource, ACCOUNTS_DB_CONFIG_FOR_TESTING}, + accounts_hash::AccountsHash, + accounts_index::AccountSecondaryIndexes, + epoch_accounts_hash::EpochAccountsHash, + }, solana_core::{ accounts_hash_verifier::AccountsHashVerifier, snapshot_packager_service::SnapshotPackagerService, @@ -14,14 +20,10 @@ use { solana_runtime::{ accounts_background_service::{ AbsRequestHandlers, AbsRequestSender, AccountsBackgroundService, - PrunedBanksRequestHandler, SnapshotRequestHandler, + PrunedBanksRequestHandler, SendDroppedBankCallback, SnapshotRequestHandler, }, - accounts_db::{self, CalcAccountsHashDataSource, ACCOUNTS_DB_CONFIG_FOR_TESTING}, - accounts_hash::AccountsHash, - accounts_index::AccountSecondaryIndexes, bank::Bank, bank_forks::BankForks, - epoch_accounts_hash::EpochAccountsHash, genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, runtime_config::RuntimeConfig, snapshot_archive_info::FullSnapshotArchiveInfo, @@ -958,14 +960,15 @@ fn test_snapshots_with_background_services( let (snapshot_package_sender, snapshot_package_receiver) = unbounded(); let bank_forks = Arc::new(RwLock::new(snapshot_test_config.bank_forks)); - let callback = bank_forks + bank_forks .read() .unwrap() .root_bank() .rc .accounts .accounts_db - .create_drop_bank_callback(pruned_banks_sender); + .enable_bank_drop_callback(); + let callback = SendDroppedBankCallback::new(pruned_banks_sender); for bank in bank_forks.read().unwrap().banks().values() { bank.set_callback(Some(Box::new(callback.clone()))); } diff --git a/genesis-utils/Cargo.toml b/genesis-utils/Cargo.toml index 223a4566f8..cffc68a9aa 100644 --- a/genesis-utils/Cargo.toml +++ b/genesis-utils/Cargo.toml @@ -11,9 +11,9 @@ edition = { workspace = true } [dependencies] log = { workspace = true } +solana-accounts-db = { workspace = true } solana-download-utils = { workspace = true } solana-rpc-client = { workspace = true } -solana-runtime = { workspace = true } solana-sdk = { workspace = true } [lib] diff --git a/genesis-utils/src/lib.rs b/genesis-utils/src/lib.rs index 7fd317567c..4add65d78f 100644 --- a/genesis-utils/src/lib.rs +++ b/genesis-utils/src/lib.rs @@ -1,8 +1,8 @@ use { log::*, + solana_accounts_db::hardened_unpack::unpack_genesis_archive, solana_download_utils::download_genesis_if_missing, solana_rpc_client::rpc_client::RpcClient, - solana_runtime::hardened_unpack::unpack_genesis_archive, solana_sdk::{ genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE}, hash::Hash, diff --git a/genesis/Cargo.toml b/genesis/Cargo.toml index a63ea89c1c..b1e57a296c 100644 --- a/genesis/Cargo.toml +++ b/genesis/Cargo.toml @@ -17,6 +17,7 @@ itertools = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_yaml = { workspace = true } +solana-accounts-db = { workspace = true } solana-clap-utils = { workspace = true } solana-cli-config = { workspace = true } solana-entry = { workspace = true } diff --git a/genesis/src/main.rs b/genesis/src/main.rs index 85136fd759..ef9785f09d 100644 --- a/genesis/src/main.rs +++ b/genesis/src/main.rs @@ -5,6 +5,7 @@ use { base64::{prelude::BASE64_STANDARD, Engine}, clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg, ArgMatches}, itertools::Itertools, + solana_accounts_db::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, solana_clap_utils::{ input_parsers::{ cluster_type_of, pubkey_of, pubkeys_of, unix_timestamp_from_rfc3339_datetime, @@ -16,7 +17,6 @@ use { solana_entry::poh::compute_hashes_per_tick, solana_genesis::{genesis_accounts::add_genesis_accounts, Base64Account}, solana_ledger::{blockstore::create_new_ledger, blockstore_options::LedgerColumnOptions}, - solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, solana_sdk::{ account::{Account, AccountSharedData, ReadableAccount, WritableAccount}, bpf_loader_upgradeable::UpgradeableLoaderState, diff --git a/geyser-plugin-manager/Cargo.toml b/geyser-plugin-manager/Cargo.toml index bf0d61a636..9b4468edda 100644 --- a/geyser-plugin-manager/Cargo.toml +++ b/geyser-plugin-manager/Cargo.toml @@ -19,6 +19,7 @@ jsonrpc-server-utils = { workspace = true } libloading = { workspace = true } log = { workspace = true } serde_json = { workspace = true } +solana-accounts-db = { workspace = true } solana-entry = { workspace = true } solana-geyser-plugin-interface = { workspace = true } solana-ledger = { workspace = true } diff --git a/geyser-plugin-manager/src/accounts_update_notifier.rs b/geyser-plugin-manager/src/accounts_update_notifier.rs index f85f2c9771..7c7e3370fc 100644 --- a/geyser-plugin-manager/src/accounts_update_notifier.rs +++ b/geyser-plugin-manager/src/accounts_update_notifier.rs @@ -2,15 +2,15 @@ use { crate::geyser_plugin_manager::GeyserPluginManager, log::*, + solana_accounts_db::{ + account_storage::meta::StoredAccountMeta, + accounts_update_notifier_interface::AccountsUpdateNotifierInterface, + }, solana_geyser_plugin_interface::geyser_plugin_interface::{ ReplicaAccountInfoV3, ReplicaAccountInfoVersions, }, solana_measure::measure::Measure, solana_metrics::*, - solana_runtime::{ - account_storage::meta::StoredAccountMeta, - accounts_update_notifier_interface::AccountsUpdateNotifierInterface, - }, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, clock::Slot, diff --git a/geyser-plugin-manager/src/block_metadata_notifier.rs b/geyser-plugin-manager/src/block_metadata_notifier.rs index 59922c9a84..2fcf409ca4 100644 --- a/geyser-plugin-manager/src/block_metadata_notifier.rs +++ b/geyser-plugin-manager/src/block_metadata_notifier.rs @@ -4,12 +4,12 @@ use { geyser_plugin_manager::GeyserPluginManager, }, log::*, + solana_accounts_db::stake_rewards::RewardInfo, solana_geyser_plugin_interface::geyser_plugin_interface::{ ReplicaBlockInfoV2, ReplicaBlockInfoVersions, }, solana_measure::measure::Measure, solana_metrics::*, - solana_runtime::bank::RewardInfo, solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey}, solana_transaction_status::{Reward, Rewards}, std::sync::{Arc, RwLock}, diff --git a/geyser-plugin-manager/src/block_metadata_notifier_interface.rs b/geyser-plugin-manager/src/block_metadata_notifier_interface.rs index 663443c6e3..9b7c34ed5c 100644 --- a/geyser-plugin-manager/src/block_metadata_notifier_interface.rs +++ b/geyser-plugin-manager/src/block_metadata_notifier_interface.rs @@ -1,5 +1,5 @@ use { - solana_runtime::bank::RewardInfo, + solana_accounts_db::stake_rewards::RewardInfo, solana_sdk::{clock::UnixTimestamp, pubkey::Pubkey}, std::sync::{Arc, RwLock}, }; diff --git a/geyser-plugin-manager/src/geyser_plugin_service.rs b/geyser-plugin-manager/src/geyser_plugin_service.rs index a64190d6ba..b8f9db4910 100644 --- a/geyser-plugin-manager/src/geyser_plugin_service.rs +++ b/geyser-plugin-manager/src/geyser_plugin_service.rs @@ -11,12 +11,12 @@ use { }, crossbeam_channel::Receiver, log::*, + solana_accounts_db::accounts_update_notifier_interface::AccountsUpdateNotifier, solana_ledger::entry_notifier_interface::EntryNotifierLock, solana_rpc::{ optimistically_confirmed_bank_tracker::SlotNotification, transaction_notifier_interface::TransactionNotifierLock, }, - solana_runtime::accounts_update_notifier_interface::AccountsUpdateNotifier, std::{ path::{Path, PathBuf}, sync::{ diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index 8f794ff6a2..fb387773c1 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -25,6 +25,7 @@ regex = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } solana-account-decoder = { workspace = true } +solana-accounts-db = { workspace = true } solana-bpf-loader-program = { workspace = true } solana-clap-utils = { workspace = true } solana-cli-output = { workspace = true } diff --git a/ledger-tool/src/args.rs b/ledger-tool/src/args.rs index 25f2a0abe5..c9db2e302f 100644 --- a/ledger-tool/src/args.rs +++ b/ledger-tool/src/args.rs @@ -1,6 +1,6 @@ use { clap::{value_t, values_t_or_exit, ArgMatches}, - solana_runtime::{ + solana_accounts_db::{ accounts_db::{AccountsDb, AccountsDbConfig, FillerAccountsConfig}, accounts_index::{AccountsIndexConfig, IndexLimitMb}, partitioned_rewards::TestPartitionedEpochRewards, diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index c4b8a073e5..0bad17855e 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -2,6 +2,7 @@ use { clap::{value_t, value_t_or_exit, values_t_or_exit, ArgMatches}, crossbeam_channel::unbounded, log::*, + solana_accounts_db::hardened_unpack::open_genesis_config, solana_core::{ accounts_hash_verifier::AccountsHashVerifier, validator::BlockVerificationMethod, }, @@ -26,7 +27,6 @@ use { PrunedBanksRequestHandler, SnapshotRequestHandler, }, bank_forks::BankForks, - hardened_unpack::open_genesis_config, snapshot_config::SnapshotConfig, snapshot_hash::StartingSnapshotHashes, snapshot_utils::{ diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index 83ee92d6ef..7ae29ed656 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -16,6 +16,10 @@ use { }, serde_json::json, solana_account_decoder::{UiAccount, UiAccountData, UiAccountEncoding}, + solana_accounts_db::{ + accounts::Accounts, accounts_db::CalcAccountsHashDataSource, accounts_index::ScanConfig, + hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + }, solana_clap_utils::{ hidden_unless_forced, input_parsers::{cluster_type_of, pubkey_of, pubkeys_of}, @@ -46,12 +50,8 @@ use { }, solana_measure::{measure, measure::Measure}, solana_runtime::{ - accounts::Accounts, - accounts_db::CalcAccountsHashDataSource, - accounts_index::ScanConfig, bank::{Bank, RewardCalculationEvent, TotalAccountsStats}, bank_forks::BankForks, - hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, runtime_config::RuntimeConfig, snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_bank_utils, @@ -2292,7 +2292,7 @@ fn main() { create_new_ledger( &output_directory, &genesis_config, - solana_runtime::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + solana_accounts_db::hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, LedgerColumnOptions::default(), ) .unwrap_or_else(|err| { diff --git a/ledger/Cargo.toml b/ledger/Cargo.toml index fbdef0bae1..e94f936d51 100644 --- a/ledger/Cargo.toml +++ b/ledger/Cargo.toml @@ -37,6 +37,7 @@ serde = { workspace = true } serde_bytes = { workspace = true } sha2 = { workspace = true } solana-account-decoder = { workspace = true } +solana-accounts-db = { workspace = true } solana-bpf-loader-program = { workspace = true } solana-cost-model = { workspace = true } solana-entry = { workspace = true } diff --git a/ledger/src/bank_forks_utils.rs b/ledger/src/bank_forks_utils.rs index 3c55cebebe..b46d950adb 100644 --- a/ledger/src/bank_forks_utils.rs +++ b/ledger/src/bank_forks_utils.rs @@ -10,9 +10,9 @@ use { use_snapshot_archives_at_startup::{self, UseSnapshotArchivesAtStartup}, }, log::*, + solana_accounts_db::accounts_update_notifier_interface::AccountsUpdateNotifier, solana_runtime::{ accounts_background_service::AbsRequestSender, - accounts_update_notifier_interface::AccountsUpdateNotifier, bank_forks::BankForks, snapshot_archive_info::{ FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter, diff --git a/ledger/src/blockstore.rs b/ledger/src/blockstore.rs index 5c794e0e9d..41d4f22406 100644 --- a/ledger/src/blockstore.rs +++ b/ledger/src/blockstore.rs @@ -32,6 +32,9 @@ use { ThreadPool, }, rocksdb::{DBRawIterator, LiveFile}, + solana_accounts_db::hardened_unpack::{ + unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + }, solana_entry::entry::{create_ticks, Entry}, solana_measure::measure::Measure, solana_metrics::{ @@ -39,10 +42,7 @@ use { poh_timing_point::{send_poh_timing_point, PohTimingSender, SlotPohTimingInfo}, }, solana_rayon_threadlimit::get_max_thread_count, - solana_runtime::{ - bank::Bank, - hardened_unpack::{unpack_genesis_archive, MAX_GENESIS_ARCHIVE_UNPACKED_SIZE}, - }, + solana_runtime::bank::Bank, solana_sdk::{ clock::{Slot, UnixTimestamp, DEFAULT_TICKS_PER_SECOND}, genesis_config::{GenesisConfig, DEFAULT_GENESIS_ARCHIVE, DEFAULT_GENESIS_FILE}, diff --git a/ledger/src/blockstore_db.rs b/ledger/src/blockstore_db.rs index c563186697..d14899ede3 100644 --- a/ledger/src/blockstore_db.rs +++ b/ledger/src/blockstore_db.rs @@ -26,7 +26,7 @@ use { WriteBatch as RWriteBatch, DB, }, serde::{de::DeserializeOwned, Serialize}, - solana_runtime::hardened_unpack::UnpackError, + solana_accounts_db::hardened_unpack::UnpackError, solana_sdk::{ clock::{Slot, UnixTimestamp}, pubkey::Pubkey, diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 29e5724bb0..654e7c2e57 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -16,6 +16,16 @@ use { rand::{seq::SliceRandom, thread_rng}, rayon::{prelude::*, ThreadPool}, scopeguard::defer, + solana_accounts_db::{ + accounts_db::{AccountShrinkThreshold, AccountsDbConfig}, + accounts_index::AccountSecondaryIndexes, + accounts_update_notifier_interface::AccountsUpdateNotifier, + epoch_accounts_hash::EpochAccountsHash, + rent_debits::RentDebits, + transaction_results::{ + TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, + }, + }, solana_cost_model::cost_model::CostModel, solana_entry::entry::{ self, create_ticks, Entry, EntrySlice, EntryType, EntryVerificationStatus, VerifyRecyclers, @@ -26,21 +36,13 @@ use { solana_rayon_threadlimit::{get_max_thread_count, get_thread_count}, solana_runtime::{ accounts_background_service::{AbsRequestSender, SnapshotRequestType}, - accounts_db::{AccountShrinkThreshold, AccountsDbConfig}, - accounts_index::AccountSecondaryIndexes, - accounts_update_notifier_interface::AccountsUpdateNotifier, bank::{Bank, TransactionBalancesSet}, bank_forks::BankForks, bank_utils, commitment::VOTE_THRESHOLD_SIZE, - epoch_accounts_hash::EpochAccountsHash, prioritization_fee_cache::PrioritizationFeeCache, - rent_debits::RentDebits, runtime_config::RuntimeConfig, transaction_batch::TransactionBatch, - transaction_results::{ - TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, - }, vote_account::VoteAccountsHashMap, vote_sender_types::ReplayVoteSender, }, diff --git a/local-cluster/Cargo.toml b/local-cluster/Cargo.toml index 05e0772a70..9c7fc6330b 100644 --- a/local-cluster/Cargo.toml +++ b/local-cluster/Cargo.toml @@ -15,6 +15,7 @@ itertools = { workspace = true } log = { workspace = true } rand = { workspace = true } rayon = { workspace = true } +solana-accounts-db = { workspace = true } solana-client = { workspace = true } solana-config-program = { workspace = true } solana-core = { workspace = true } diff --git a/local-cluster/src/local_cluster.rs b/local-cluster/src/local_cluster.rs index 0ff56d4419..d180a4abaf 100644 --- a/local-cluster/src/local_cluster.rs +++ b/local-cluster/src/local_cluster.rs @@ -6,6 +6,7 @@ use { }, itertools::izip, log::*, + solana_accounts_db::accounts_db::create_accounts_run_and_snapshot_dirs, solana_client::{connection_cache::ConnectionCache, thin_client::ThinClient}, solana_core::{ consensus::tower_storage::FileTowerStorage, @@ -23,7 +24,6 @@ use { ValidatorVoteKeypairs, }, snapshot_config::SnapshotConfig, - snapshot_utils::create_accounts_run_and_snapshot_dirs, }, solana_sdk::{ account::{Account, AccountSharedData}, diff --git a/local-cluster/tests/common/mod.rs b/local-cluster/tests/common/mod.rs index 0926156277..03e87ccd98 100644 --- a/local-cluster/tests/common/mod.rs +++ b/local-cluster/tests/common/mod.rs @@ -1,6 +1,7 @@ #![allow(clippy::integer_arithmetic)] use { log::*, + solana_accounts_db::accounts_db::create_accounts_run_and_snapshot_dirs, solana_core::{ consensus::{tower_storage::FileTowerStorage, Tower, SWITCH_FORK_THRESHOLD}, validator::{is_snapshot_config_valid, ValidatorConfig}, @@ -21,7 +22,6 @@ use { solana_rpc_client::rpc_client::RpcClient, solana_runtime::{ snapshot_bank_utils::DISABLED_SNAPSHOT_ARCHIVE_INTERVAL, snapshot_config::SnapshotConfig, - snapshot_utils::create_accounts_run_and_snapshot_dirs, }, solana_sdk::{ account::AccountSharedData, diff --git a/local-cluster/tests/local_cluster.rs b/local-cluster/tests/local_cluster.rs index 506c75f541..0885082530 100644 --- a/local-cluster/tests/local_cluster.rs +++ b/local-cluster/tests/local_cluster.rs @@ -6,6 +6,9 @@ use { gag::BufferRedirect, log::*, serial_test::serial, + solana_accounts_db::{ + accounts_db::create_accounts_run_and_snapshot_dirs, hardened_unpack::open_genesis_config, + }, solana_client::thin_client::ThinClient, solana_core::{ consensus::{ @@ -45,12 +48,11 @@ use { }, solana_runtime::{ commitment::VOTE_THRESHOLD_SIZE, - hardened_unpack::open_genesis_config, snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_bank_utils, snapshot_config::SnapshotConfig, snapshot_package::SnapshotType, - snapshot_utils::{self, create_accounts_run_and_snapshot_dirs}, + snapshot_utils::{self}, vote_parser, }, solana_sdk::{ diff --git a/merkle-root-bench/Cargo.toml b/merkle-root-bench/Cargo.toml index 5d58a7e887..1f2c1ea23d 100644 --- a/merkle-root-bench/Cargo.toml +++ b/merkle-root-bench/Cargo.toml @@ -11,9 +11,9 @@ edition = { workspace = true } [dependencies] clap = { workspace = true } log = { workspace = true } +solana-accounts-db = { workspace = true } solana-logger = { workspace = true } solana-measure = { workspace = true } -solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-version = { workspace = true } diff --git a/merkle-root-bench/src/main.rs b/merkle-root-bench/src/main.rs index 87fa93b31f..0b93360c4a 100644 --- a/merkle-root-bench/src/main.rs +++ b/merkle-root-bench/src/main.rs @@ -1,8 +1,8 @@ extern crate log; use { clap::{crate_description, crate_name, value_t, App, Arg}, + solana_accounts_db::accounts_hash::AccountsHasher, solana_measure::measure::Measure, - solana_runtime::accounts_hash::AccountsHasher, solana_sdk::{hash::Hash, pubkey::Pubkey}, }; diff --git a/program-test/Cargo.toml b/program-test/Cargo.toml index 935680593e..07e05e88fb 100644 --- a/program-test/Cargo.toml +++ b/program-test/Cargo.toml @@ -17,6 +17,7 @@ chrono-humanize = { workspace = true } crossbeam-channel = { workspace = true } log = { workspace = true } serde = { workspace = true } +solana-accounts-db = { workspace = true } solana-banks-client = { workspace = true } solana-banks-interface = { workspace = true } solana-banks-server = { workspace = true } diff --git a/program-test/src/lib.rs b/program-test/src/lib.rs index 03d67d8faa..e6e731db9c 100644 --- a/program-test/src/lib.rs +++ b/program-test/src/lib.rs @@ -8,6 +8,7 @@ use { base64::{prelude::BASE64_STANDARD, Engine}, chrono_humanize::{Accuracy, HumanTime, Tense}, log::*, + solana_accounts_db::epoch_accounts_hash::EpochAccountsHash, solana_banks_client::start_client, solana_banks_server::banks_server::start_local_server, solana_bpf_loader_program::serialization::serialize_parameters, @@ -20,7 +21,6 @@ use { bank::Bank, bank_forks::BankForks, commitment::BlockCommitmentCache, - epoch_accounts_hash::EpochAccountsHash, genesis_utils::{create_genesis_config_with_leader_ex, GenesisConfigInfo}, runtime_config::RuntimeConfig, }, @@ -1118,7 +1118,7 @@ impl ProgramTestContext { &Pubkey::default(), pre_warp_slot, // some warping tests cannot use the append vecs because of the sequence of adding roots and flushing - solana_runtime::accounts_db::CalcAccountsHashDataSource::IndexForTests, + solana_accounts_db::accounts_db::CalcAccountsHashDataSource::IndexForTests, )) }; diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index cb0a96bf3a..b9a523ffa5 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -4527,6 +4527,76 @@ dependencies = [ "zstd", ] +[[package]] +name = "solana-accounts-db" +version = "1.17.0" +dependencies = [ + "arrayref", + "bincode", + "blake3", + "bv", + "bytemuck", + "byteorder 1.4.3", + "bzip2", + "crossbeam-channel", + "dashmap", + "dir-diff", + "flate2", + "fnv", + "fs-err", + "im", + "index_list", + "itertools", + "lazy_static", + "log", + "lru", + "lz4", + "memmap2", + "modular-bitfield", + "num-derive", + "num-traits", + "num_cpus", + "num_enum 0.6.1", + "once_cell", + "ouroboros", + "percentage", + "rand 0.7.3", + "rayon", + "regex", + "rustc_version", + "serde", + "serde_derive", + "siphasher", + "solana-address-lookup-table-program", + "solana-bpf-loader-program", + "solana-bucket-map", + "solana-compute-budget-program", + "solana-config-program", + "solana-cost-model", + "solana-frozen-abi", + "solana-frozen-abi-macro", + "solana-loader-v4-program", + "solana-measure", + "solana-metrics", + "solana-perf", + "solana-program-runtime", + "solana-rayon-threadlimit", + "solana-sdk", + "solana-stake-program", + "solana-system-program", + "solana-vote-program", + "solana-zk-token-proof-program", + "solana-zk-token-sdk", + "static_assertions", + "strum", + "strum_macros", + "symlink", + "tar", + "tempfile", + "thiserror", + "zstd", +] + [[package]] name = "solana-address-lookup-table-program" version = "1.17.0" @@ -4577,6 +4647,7 @@ dependencies = [ "bincode", "crossbeam-channel", "futures 0.3.28", + "solana-accounts-db", "solana-banks-interface", "solana-client", "solana-runtime", @@ -4799,6 +4870,7 @@ dependencies = [ "rustc_version", "serde", "serde_derive", + "solana-accounts-db", "solana-address-lookup-table-program", "solana-bloom", "solana-client", @@ -4960,9 +5032,9 @@ name = "solana-genesis-utils" version = "1.17.0" dependencies = [ "log", + "solana-accounts-db", "solana-download-utils", "solana-rpc-client", - "solana-runtime", "solana-sdk", ] @@ -4988,6 +5060,7 @@ dependencies = [ "libloading", "log", "serde_json", + "solana-accounts-db", "solana-entry", "solana-geyser-plugin-interface", "solana-ledger", @@ -5079,6 +5152,7 @@ dependencies = [ "serde_bytes", "sha2 0.10.7", "solana-account-decoder", + "solana-accounts-db", "solana-bpf-loader-program", "solana-cost-model", "solana-entry", @@ -5309,6 +5383,7 @@ dependencies = [ "crossbeam-channel", "log", "serde", + "solana-accounts-db", "solana-banks-client", "solana-banks-interface", "solana-banks-server", @@ -5419,6 +5494,7 @@ dependencies = [ "serde_json", "soketto", "solana-account-decoder", + "solana-accounts-db", "solana-client", "solana-entry", "solana-faucet", @@ -5543,6 +5619,7 @@ dependencies = [ "serde", "serde_derive", "siphasher", + "solana-accounts-db", "solana-address-lookup-table-program", "solana-bpf-loader-program", "solana-bucket-map", @@ -5585,6 +5662,7 @@ dependencies = [ "miow", "net2", "solana-account-decoder", + "solana-accounts-db", "solana-bpf-loader-program", "solana-cli-output", "solana-ledger", @@ -6155,6 +6233,7 @@ dependencies = [ "log", "serde_derive", "serde_json", + "solana-accounts-db", "solana-cli-output", "solana-client", "solana-core", @@ -6310,6 +6389,7 @@ dependencies = [ "serde_json", "serde_yaml", "signal-hook", + "solana-accounts-db", "solana-clap-utils", "solana-cli-config", "solana-core", diff --git a/programs/sbf/Cargo.toml b/programs/sbf/Cargo.toml index 0d554e6e62..9187a87e92 100644 --- a/programs/sbf/Cargo.toml +++ b/programs/sbf/Cargo.toml @@ -26,6 +26,7 @@ serde = "1.0.112" serde_json = "1.0.56" solana_rbpf = "=0.6.0" solana-account-decoder = { path = "../../account-decoder", version = "=1.17.0" } +solana-accounts-db = { path = "../../accounts-db", version = "=1.17.0" } solana-address-lookup-table-program = { path = "../../programs/address-lookup-table", version = "=1.17.0" } solana-bpf-loader-program = { path = "../bpf_loader", version = "=1.17.0" } solana-cli-output = { path = "../../cli-output", version = "=1.17.0" } @@ -84,6 +85,7 @@ log = { workspace = true } miow = { workspace = true } net2 = { workspace = true } solana-account-decoder = { workspace = true } +solana-accounts-db = { workspace = true } solana-bpf-loader-program = { workspace = true } solana-cli-output = { workspace = true } solana-logger = { workspace = true } diff --git a/programs/sbf/tests/programs.rs b/programs/sbf/tests/programs.rs index 002830a593..8638b2b14d 100644 --- a/programs/sbf/tests/programs.rs +++ b/programs/sbf/tests/programs.rs @@ -15,6 +15,10 @@ use { solana_account_decoder::parse_bpf_loader::{ parse_bpf_upgradeable_loader, BpfUpgradeableLoaderAccountType, }, + solana_accounts_db::transaction_results::{ + DurableNonceFee, InnerInstruction, TransactionExecutionDetails, TransactionExecutionResult, + TransactionResults, + }, solana_ledger::token_balances::collect_token_balances, solana_program_runtime::{compute_budget::ComputeBudget, timings::ExecuteTimings}, solana_rbpf::vm::ContextObject, @@ -25,10 +29,6 @@ use { load_upgradeable_buffer, load_upgradeable_program, set_upgrade_authority, upgrade_program, }, - transaction_results::{ - DurableNonceFee, InnerInstruction, TransactionExecutionDetails, - TransactionExecutionResult, TransactionResults, - }, }, solana_sbf_rust_invoke::instructions::*, solana_sbf_rust_realloc::instructions::*, diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index c94c40fa34..92bc7710c2 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -30,6 +30,7 @@ serde_derive = { workspace = true } serde_json = { workspace = true } soketto = { workspace = true } solana-account-decoder = { workspace = true } +solana-accounts-db = { workspace = true } solana-client = { workspace = true } solana-entry = { workspace = true } solana-faucet = { workspace = true } diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 7ac0dac986..020f4193a2 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -1,5 +1,4 @@ //! The `rpc` module implements the Solana RPC interface. - use { crate::{ max_slots::MaxSlots, optimistically_confirmed_bank_tracker::OptimisticallyConfirmedBank, @@ -14,6 +13,12 @@ use { parse_token::{is_known_spl_token_id, token_amount_to_ui_amount, UiTokenAmount}, UiAccount, UiAccountEncoding, UiDataSliceConfig, MAX_BASE58_BYTES, }, + solana_accounts_db::{ + accounts::AccountAddressFilter, + accounts_index::{AccountIndex, AccountSecondaryIndexes, IndexKey, ScanConfig}, + inline_spl_token::{SPL_TOKEN_ACCOUNT_MINT_OFFSET, SPL_TOKEN_ACCOUNT_OWNER_OFFSET}, + inline_spl_token_2022::{self, ACCOUNTTYPE_ACCOUNT}, + }, solana_client::connection_cache::{ConnectionCache, Protocol}, solana_entry::entry::Entry, solana_faucet::faucet::request_airdrop_transaction, @@ -42,13 +47,9 @@ use { response::{Response as RpcResponse, *}, }, solana_runtime::{ - accounts::AccountAddressFilter, - accounts_index::{AccountIndex, AccountSecondaryIndexes, IndexKey, ScanConfig}, bank::{Bank, TransactionSimulationResult}, bank_forks::BankForks, commitment::{BlockCommitmentArray, BlockCommitmentCache, CommitmentSlots}, - inline_spl_token::{SPL_TOKEN_ACCOUNT_MINT_OFFSET, SPL_TOKEN_ACCOUNT_OWNER_OFFSET}, - inline_spl_token_2022::{self, ACCOUNTTYPE_ACCOUNT}, non_circulating_supply::calculate_non_circulating_supply, prioritization_fee_cache::PrioritizationFeeCache, snapshot_config::SnapshotConfig, @@ -4632,6 +4633,7 @@ pub mod tests { jsonrpc_core::{futures, ErrorCode, MetaIoHandler, Output, Response, Value}, jsonrpc_core_client::transports::local, serde::de::DeserializeOwned, + solana_accounts_db::{inline_spl_token, inline_spl_token_2022}, solana_address_lookup_table_program::state::{AddressLookupTable, LookupTableMeta}, solana_entry::entry::next_versioned_entry, solana_gossip::socketaddr, @@ -4650,8 +4652,7 @@ pub mod tests { }, solana_runtime::{ accounts_background_service::AbsRequestSender, bank::BankTestConfig, - commitment::BlockCommitment, inline_spl_token, - non_circulating_supply::non_circulating_accounts, + commitment::BlockCommitment, non_circulating_supply::non_circulating_accounts, }, solana_sdk::{ account::{Account, WritableAccount}, diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index 25b03e8b77..c4c619a1a3 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -2,11 +2,11 @@ use { crate::transaction_notifier_interface::TransactionNotifierLock, crossbeam_channel::{Receiver, RecvTimeoutError}, itertools::izip, + solana_accounts_db::transaction_results::{DurableNonceFee, TransactionExecutionDetails}, solana_ledger::{ blockstore::Blockstore, blockstore_processor::{TransactionStatusBatch, TransactionStatusMessage}, }, - solana_runtime::transaction_results::{DurableNonceFee, TransactionExecutionDetails}, solana_transaction_status::{ extract_and_fmt_memos, InnerInstruction, InnerInstructions, Reward, TransactionStatusMeta, }, @@ -225,12 +225,12 @@ pub(crate) mod tests { crossbeam_channel::unbounded, dashmap::DashMap, solana_account_decoder::parse_token::token_amount_to_ui_amount, - solana_ledger::{genesis_utils::create_genesis_config, get_tmp_ledger_path}, - solana_runtime::{ - bank::{Bank, TransactionBalancesSet}, + solana_accounts_db::{ nonce_info::{NonceFull, NoncePartial}, rent_debits::RentDebits, }, + solana_ledger::{genesis_utils::create_genesis_config, get_tmp_ledger_path}, + solana_runtime::bank::{Bank, TransactionBalancesSet}, solana_sdk::{ account_utils::StateMut, clock::Slot, diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index f2da3dce04..3fa4ad3dd7 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -45,6 +45,7 @@ regex = { workspace = true } serde = { workspace = true, features = ["rc"] } serde_derive = { workspace = true } siphasher = { workspace = true } +solana-accounts-db = { workspace = true } solana-address-lookup-table-program = { workspace = true } solana-bpf-loader-program = { workspace = true } solana-bucket-map = { workspace = true } @@ -84,6 +85,7 @@ ed25519-dalek = { workspace = true } libsecp256k1 = { workspace = true } memoffset = { workspace = true } rand_chacha = { workspace = true } +solana-accounts-db = { workspace = true, features = ["dev-context-only-utils"] } solana-logger = { workspace = true } # See order-crates-for-publishing.py for using this unusual `path = "."` solana-runtime = { path = ".", features = ["dev-context-only-utils"] } diff --git a/runtime/benches/accounts.rs b/runtime/benches/accounts.rs index 20f1ed3e81..479c3bc8f2 100644 --- a/runtime/benches/accounts.rs +++ b/runtime/benches/accounts.rs @@ -7,7 +7,7 @@ use { dashmap::DashMap, rand::Rng, rayon::iter::{IntoParallelRefIterator, ParallelIterator}, - solana_runtime::{ + solana_accounts_db::{ accounts::{AccountAddressFilter, Accounts}, accounts_db::{ test_utils::create_test_accounts, AccountShrinkThreshold, @@ -15,9 +15,9 @@ use { }, accounts_index::{AccountSecondaryIndexes, ScanConfig}, ancestors::Ancestors, - bank::*, rent_collector::RentCollector, }, + solana_runtime::bank::*, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, genesis_config::{create_genesis_config, ClusterType}, diff --git a/runtime/benches/accounts_index.rs b/runtime/benches/accounts_index.rs index 336ec89c5a..693ca62b45 100644 --- a/runtime/benches/accounts_index.rs +++ b/runtime/benches/accounts_index.rs @@ -4,7 +4,7 @@ extern crate test; use { rand::{thread_rng, Rng}, - solana_runtime::{ + solana_accounts_db::{ account_info::AccountInfo, accounts_index::{ AccountSecondaryIndexes, AccountsIndex, UpsertReclaim, diff --git a/runtime/benches/append_vec.rs b/runtime/benches/append_vec.rs index 535395d0c3..0856ac4007 100644 --- a/runtime/benches/append_vec.rs +++ b/runtime/benches/append_vec.rs @@ -3,7 +3,7 @@ extern crate test; use { rand::{thread_rng, Rng}, - solana_runtime::{ + solana_accounts_db::{ account_storage::meta::{ StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredMeta, }, diff --git a/runtime/src/accounts_background_service.rs b/runtime/src/accounts_background_service.rs index 40d0f7e2b8..02724813b1 100644 --- a/runtime/src/accounts_background_service.rs +++ b/runtime/src/accounts_background_service.rs @@ -5,8 +5,6 @@ mod stats; use { crate::{ - accounts_db::CalcAccountsHashDataSource, - accounts_hash::CalcAccountsHashConfig, bank::{Bank, BankSlotDelta, DropCallback}, bank_forks::BankForks, snapshot_bank_utils, @@ -18,6 +16,9 @@ use { log::*, rand::{thread_rng, Rng}, rayon::iter::{IntoParallelIterator, ParallelIterator}, + solana_accounts_db::{ + accounts_db::CalcAccountsHashDataSource, accounts_hash::CalcAccountsHashConfig, + }, solana_measure::measure::Measure, solana_sdk::clock::{BankId, Slot}, stats::StatsManager, @@ -41,7 +42,8 @@ const CLEAN_INTERVAL_BLOCKS: u64 = 100; // this would drop MAX_RECYCLE_STORES mmaps at once in the worst case... // (Anyway, the dropping part is outside the AccountsDb::recycle_stores lock // and dropped in this AccountsBackgroundServe, so this shouldn't matter much) -const RECYCLE_STORE_EXPIRATION_INTERVAL_SECS: u64 = crate::accounts_db::EXPIRATION_TTL_SECONDS / 3; +const RECYCLE_STORE_EXPIRATION_INTERVAL_SECS: u64 = + solana_accounts_db::accounts_db::EXPIRATION_TTL_SECONDS / 3; pub type SnapshotRequestSender = Sender; pub type SnapshotRequestReceiver = Receiver; @@ -704,13 +706,15 @@ impl AccountsBackgroundService { let (pruned_banks_sender, pruned_banks_receiver) = crossbeam_channel::unbounded(); { let root_bank = bank_forks.read().unwrap().root_bank(); - root_bank.set_callback(Some(Box::new( - root_bank - .rc - .accounts - .accounts_db - .create_drop_bank_callback(pruned_banks_sender), - ))); + + root_bank + .rc + .accounts + .accounts_db + .enable_bank_drop_callback(); + root_bank.set_callback(Some(Box::new(SendDroppedBankCallback::new( + pruned_banks_sender, + )))); } pruned_banks_receiver } @@ -790,11 +794,9 @@ fn cmp_requests_by_priority( mod test { use { super::*, - crate::{ - bank::epoch_accounts_hash_utils, epoch_accounts_hash::EpochAccountsHash, - genesis_utils::create_genesis_config, - }, + crate::{bank::epoch_accounts_hash_utils, genesis_utils::create_genesis_config}, crossbeam_channel::unbounded, + solana_accounts_db::epoch_accounts_hash::EpochAccountsHash, solana_sdk::{ account::AccountSharedData, epoch_schedule::EpochSchedule, hash::Hash, pubkey::Pubkey, }, diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 8d3a5e8610..c298d104a2 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -38,6 +38,35 @@ use solana_sdk::recent_blockhashes_account; pub use solana_sdk::reward_type::RewardType; use { crate::{ + bank::metrics::*, + builtins::{BuiltinPrototype, BUILTINS}, + epoch_rewards_hasher::hash_rewards_into_partitions, + epoch_stakes::{EpochStakes, NodeVoteAccounts}, + runtime_config::RuntimeConfig, + serde_snapshot::BankIncrementalSnapshotPersistence, + snapshot_hash::SnapshotHash, + stake_account::StakeAccount, + stake_history::StakeHistory, + stake_weighted_timestamp::{ + calculate_stake_weighted_timestamp, MaxAllowableDrift, + MAX_ALLOWABLE_DRIFT_PERCENTAGE_FAST, MAX_ALLOWABLE_DRIFT_PERCENTAGE_SLOW_V2, + }, + stakes::{InvalidCacheEntryReason, Stakes, StakesCache, StakesEnum}, + status_cache::{SlotDelta, StatusCache}, + transaction_batch::TransactionBatch, + vote_account::{VoteAccount, VoteAccounts, VoteAccountsHashMap}, + }, + byteorder::{ByteOrder, LittleEndian}, + dashmap::{DashMap, DashSet}, + itertools::izip, + log::*, + percentage::Percentage, + rayon::{ + iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}, + slice::ParallelSlice, + ThreadPool, ThreadPoolBuilder, + }, + solana_accounts_db::{ account_overrides::AccountOverrides, account_rent_state::RentState, accounts::{ @@ -54,48 +83,21 @@ use { accounts_partition::{self, Partition, PartitionIndex}, accounts_update_notifier_interface::AccountsUpdateNotifier, ancestors::{Ancestors, AncestorsForSerialization}, - bank::metrics::*, blockhash_queue::BlockhashQueue, - builtins::{BuiltinPrototype, BUILTINS}, epoch_accounts_hash::EpochAccountsHash, - epoch_rewards_hasher::hash_rewards_into_partitions, - epoch_stakes::{EpochStakes, NodeVoteAccounts}, nonce_info::{NonceInfo, NoncePartial}, partitioned_rewards::PartitionedEpochRewardsConfig, rent_collector::{CollectedInfo, RentCollector}, rent_debits::RentDebits, - runtime_config::RuntimeConfig, - serde_snapshot::BankIncrementalSnapshotPersistence, - snapshot_hash::SnapshotHash, sorted_storages::SortedStorages, - stake_account::StakeAccount, - stake_history::StakeHistory, - stake_rewards::StakeReward, - stake_weighted_timestamp::{ - calculate_stake_weighted_timestamp, MaxAllowableDrift, - MAX_ALLOWABLE_DRIFT_PERCENTAGE_FAST, MAX_ALLOWABLE_DRIFT_PERCENTAGE_SLOW_V2, - }, - stakes::{InvalidCacheEntryReason, Stakes, StakesCache, StakesEnum}, - status_cache::{SlotDelta, StatusCache}, + stake_rewards::{RewardInfo, StakeReward}, storable_accounts::StorableAccounts, - transaction_batch::TransactionBatch, transaction_error_metrics::TransactionErrorMetrics, transaction_results::{ inner_instructions_list_from_instruction_trace, DurableNonceFee, TransactionCheckResult, TransactionExecutionDetails, TransactionExecutionResult, TransactionResults, }, - vote_account::{VoteAccount, VoteAccounts, VoteAccountsHashMap}, - }, - byteorder::{ByteOrder, LittleEndian}, - dashmap::{DashMap, DashSet}, - itertools::izip, - log::*, - percentage::Percentage, - rayon::{ - iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}, - slice::ParallelSlice, - ThreadPool, ThreadPoolBuilder, }, solana_bpf_loader_program::syscalls::create_program_runtime_environment, solana_cost_model::cost_tracker::CostTracker, @@ -617,17 +619,6 @@ pub trait DropCallback: fmt::Debug { fn clone_box(&self) -> Box; } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, AbiExample, Clone, Copy)] -pub struct RewardInfo { - pub reward_type: RewardType, - /// Reward amount - pub lamports: i64, - /// Account balance in lamports after `lamports` was applied - pub post_balance: u64, - /// Vote account commission when the reward was credited, only present for voting and staking rewards - pub commission: Option, -} - #[derive(Debug, Default)] pub struct OptionalDropCallback(Option>); @@ -932,29 +923,6 @@ pub struct CommitTransactionCounts { pub signature_count: u64, } -/// allow [StakeReward] to be passed to `StoreAccounts` directly without copies or vec construction -impl<'a> StorableAccounts<'a, AccountSharedData> for (Slot, &'a [StakeReward], IncludeSlotInHash) { - fn pubkey(&self, index: usize) -> &Pubkey { - &self.1[index].stake_pubkey - } - fn account(&self, index: usize) -> &AccountSharedData { - &self.1[index].stake_account - } - fn slot(&self, _index: usize) -> Slot { - // per-index slot is not unique per slot when per-account slot is not included in the source data - self.target_slot() - } - fn target_slot(&self) -> Slot { - self.0 - } - fn len(&self) -> usize { - self.1.len() - } - fn include_slot_in_hash(&self) -> IncludeSlotInHash { - self.2 - } -} - impl WorkingSlot for Bank { fn current_slot(&self) -> Slot { self.slot @@ -1282,7 +1250,7 @@ impl Bank { 1 } else { const MAX_FACTOR_OF_REWARD_BLOCKS_IN_EPOCH: u64 = 10; - let num_chunks = crate::accounts_hash::AccountsHasher::div_ceil( + let num_chunks = solana_accounts_db::accounts_hash::AccountsHasher::div_ceil( total_stake_accounts, self.partitioned_rewards_stake_account_stores_per_block() as usize, ) as u64; @@ -6067,7 +6035,7 @@ impl Bank { // divide the range into num_threads smaller ranges and process in parallel // Note that 'pubkey_range_from_partition' cannot easily be re-used here to break the range smaller. // It has special handling of 0..0 and partition_count changes affect all ranges unevenly. - let num_threads = crate::accounts_db::quarter_thread_count() as u64; + let num_threads = solana_accounts_db::accounts_db::quarter_thread_count() as u64; let sz = std::mem::size_of::(); let start_prefix = accounts_partition::prefix_from_pubkey(subrange_full.start()); let end_prefix_inclusive = accounts_partition::prefix_from_pubkey(subrange_full.end()); diff --git a/runtime/src/bank/serde_snapshot.rs b/runtime/src/bank/serde_snapshot.rs index ecec8e4aba..d187c7e94a 100644 --- a/runtime/src/bank/serde_snapshot.rs +++ b/runtime/src/bank/serde_snapshot.rs @@ -2,19 +2,10 @@ mod tests { use { crate::{ - account_storage::{AccountStorageMap, AccountStorageReference}, - accounts_db::{ - get_temp_accounts_paths, AccountShrinkThreshold, AccountStorageEntry, AccountsDb, - AtomicAppendVecId, - }, - accounts_file::{AccountsFile, AccountsFileError}, - accounts_hash::{AccountsDeltaHash, AccountsHash}, - accounts_index::AccountSecondaryIndexes, bank::{ epoch_accounts_hash_utils, Bank, BankTestConfig, EpochRewardStatus, StartBlockHeightAndRewards, }, - epoch_accounts_hash::EpochAccountsHash, genesis_utils::{activate_all_features, activate_feature}, runtime_config::RuntimeConfig, serde_snapshot::{ @@ -26,9 +17,20 @@ mod tests { self, create_tmp_accounts_dir_for_tests, get_storages_to_serialize, ArchiveFormat, StorageAndNextAppendVecId, BANK_SNAPSHOT_PRE_FILENAME_EXTENSION, }, - stake_rewards::StakeReward, status_cache::StatusCache, }, + solana_accounts_db::{ + account_storage::{AccountStorageMap, AccountStorageReference}, + accounts_db::{ + get_temp_accounts_paths, AccountShrinkThreshold, AccountStorageEntry, AccountsDb, + AtomicAppendVecId, + }, + accounts_file::{AccountsFile, AccountsFileError}, + accounts_hash::{AccountsDeltaHash, AccountsHash}, + accounts_index::AccountSecondaryIndexes, + epoch_accounts_hash::EpochAccountsHash, + stake_rewards::StakeReward, + }, solana_sdk::{ epoch_schedule::EpochSchedule, feature_set, @@ -269,7 +271,7 @@ mod tests { None, AccountShrinkThreshold::default(), false, - Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), + Some(solana_accounts_db::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), None, Arc::default(), ) @@ -404,7 +406,7 @@ mod tests { None, AccountShrinkThreshold::default(), false, - Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), + Some(solana_accounts_db::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), None, Arc::default(), ) @@ -496,7 +498,7 @@ mod tests { false, false, false, - Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), + Some(solana_accounts_db::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), None, Arc::default(), ) @@ -590,7 +592,7 @@ mod tests { None, AccountShrinkThreshold::default(), false, - Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), + Some(solana_accounts_db::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), None, Arc::default(), ) @@ -612,7 +614,7 @@ mod tests { // This some what long test harness is required to freeze the ABI of // Bank's serialization due to versioned nature - #[frozen_abi(digest = "A99zFXvqYm88n6EbtEFbroDbuFNnhw4K7AmqMh2wjJmh")] + #[frozen_abi(digest = "5G71eC1ofQ6pqgeQLb8zaK4EQCncs5Rs51rfmMAvtF8U")] #[derive(Serialize, AbiExample)] pub struct BankAbiTestWrapperNewer { #[serde(serialize_with = "wrapper_newer")] diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index 9882a48d9e..e061cbac20 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -7,14 +7,7 @@ use { *, }, crate::{ - accounts::{AccountAddressFilter, RewardInterval}, accounts_background_service::{PrunedBanksRequestHandler, SendDroppedBankCallback}, - accounts_db::{AccountShrinkThreshold, DEFAULT_ACCOUNTS_SHRINK_RATIO}, - accounts_index::{ - AccountIndex, AccountSecondaryIndexes, IndexKey, ScanConfig, ScanError, ITER_BATCH_SIZE, - }, - accounts_partition::{self, PartitionIndex, RentPayingAccountsByPartition}, - ancestors::Ancestors, bank_client::BankClient, epoch_rewards_hasher::hash_rewards_into_partitions, genesis_utils::{ @@ -22,18 +15,27 @@ use { create_genesis_config_with_leader, create_genesis_config_with_vote_accounts, genesis_sysvar_and_builtin_program_lamports, GenesisConfigInfo, ValidatorVoteKeypairs, }, - inline_spl_token, - nonce_info::NonceFull, - partitioned_rewards::TestPartitionedEpochRewards, - rent_collector::RENT_EXEMPT_RENT_EPOCH, status_cache::MAX_CACHE_ENTRIES, - transaction_error_metrics::TransactionErrorMetrics, }, crossbeam_channel::{bounded, unbounded}, itertools::Itertools, rand::Rng, rayon::ThreadPoolBuilder, serde::{Deserialize, Serialize}, + solana_accounts_db::{ + accounts::{AccountAddressFilter, RewardInterval}, + accounts_db::{AccountShrinkThreshold, DEFAULT_ACCOUNTS_SHRINK_RATIO}, + accounts_index::{ + AccountIndex, AccountSecondaryIndexes, IndexKey, ScanConfig, ScanError, ITER_BATCH_SIZE, + }, + accounts_partition::{self, PartitionIndex, RentPayingAccountsByPartition}, + ancestors::Ancestors, + inline_spl_token, + nonce_info::NonceFull, + partitioned_rewards::TestPartitionedEpochRewards, + rent_collector::RENT_EXEMPT_RENT_EPOCH, + transaction_error_metrics::TransactionErrorMetrics, + }, solana_logger, solana_program_runtime::{ compute_budget::{self, ComputeBudget, MAX_COMPUTE_UNIT_LIMIT}, @@ -123,52 +125,6 @@ use { test_case::test_case, }; -impl StakeReward { - pub fn new_random() -> Self { - let mut rng = rand::thread_rng(); - - let rent = Rent::free(); - - let validator_pubkey = solana_sdk::pubkey::new_rand(); - let validator_stake_lamports = 20; - let validator_staking_keypair = Keypair::new(); - let validator_voting_keypair = Keypair::new(); - - let validator_vote_account = vote_state::create_account( - &validator_voting_keypair.pubkey(), - &validator_pubkey, - 10, - validator_stake_lamports, - ); - - let validator_stake_account = stake_state::create_account( - &validator_staking_keypair.pubkey(), - &validator_voting_keypair.pubkey(), - &validator_vote_account, - &rent, - validator_stake_lamports, - ); - - Self { - stake_pubkey: Pubkey::new_unique(), - stake_reward_info: RewardInfo { - reward_type: RewardType::Staking, - lamports: rng.gen_range(1, 200), - post_balance: 0, /* unused atm */ - commission: None, /* unused atm */ - }, - - stake_account: validator_stake_account, - } - } - - pub fn credit(&mut self, amount: u64) { - self.stake_reward_info.lamports = amount as i64; - self.stake_reward_info.post_balance += amount; - self.stake_account.checked_add_lamports(amount).unwrap(); - } -} - impl VoteReward { pub fn new_random() -> Self { let mut rng = rand::thread_rng(); diff --git a/runtime/src/bank/transaction_account_state_info.rs b/runtime/src/bank/transaction_account_state_info.rs index 45f5d59e4c..11e6d540d9 100644 --- a/runtime/src/bank/transaction_account_state_info.rs +++ b/runtime/src/bank/transaction_account_state_info.rs @@ -1,8 +1,6 @@ use { - crate::{ - account_rent_state::{check_rent_state, RentState}, - bank::Bank, - }, + crate::bank::Bank, + solana_accounts_db::account_rent_state::{check_rent_state, RentState}, solana_sdk::{ account::ReadableAccount, message::SanitizedMessage, diff --git a/runtime/src/bank_forks.rs b/runtime/src/bank_forks.rs index 93307df2f4..7f377d9672 100644 --- a/runtime/src/bank_forks.rs +++ b/runtime/src/bank_forks.rs @@ -675,11 +675,11 @@ mod tests { super::*, crate::{ bank::test_utils::update_vote_account_timestamp, - epoch_accounts_hash::EpochAccountsHash, genesis_utils::{ create_genesis_config, create_genesis_config_with_leader, GenesisConfigInfo, }, }, + solana_accounts_db::epoch_accounts_hash::EpochAccountsHash, solana_sdk::{ clock::UnixTimestamp, epoch_schedule::EpochSchedule, diff --git a/runtime/src/bank_utils.rs b/runtime/src/bank_utils.rs index 38a92ef019..0eeb750bf4 100644 --- a/runtime/src/bank_utils.rs +++ b/runtime/src/bank_utils.rs @@ -2,10 +2,10 @@ use { crate::{ bank::Bank, genesis_utils::{self, GenesisConfigInfo, ValidatorVoteKeypairs}, - transaction_results::TransactionResults, vote_parser, vote_sender_types::ReplayVoteSender, }, + solana_accounts_db::transaction_results::TransactionResults, solana_sdk::{pubkey::Pubkey, signature::Signer, transaction::SanitizedTransaction}, }; diff --git a/runtime/src/epoch_rewards_hasher.rs b/runtime/src/epoch_rewards_hasher.rs index 22dedb2d72..f03f00f5e6 100644 --- a/runtime/src/epoch_rewards_hasher.rs +++ b/runtime/src/epoch_rewards_hasher.rs @@ -64,7 +64,7 @@ pub(crate) fn hash_rewards_into_partitions( mod tests { use { super::*, - crate::stake_rewards::StakeReward, + solana_accounts_db::stake_rewards::StakeReward, std::{collections::HashMap, ops::RangeInclusive}, }; diff --git a/runtime/src/genesis_utils.rs b/runtime/src/genesis_utils.rs index bc9412a1ad..19d83aede1 100644 --- a/runtime/src/genesis_utils.rs +++ b/runtime/src/genesis_utils.rs @@ -1,5 +1,5 @@ use { - crate::inline_spl_token, + solana_accounts_db::inline_spl_token, solana_sdk::{ account::{Account, AccountSharedData}, feature::{self, Feature}, diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index aed9467dcb..7978048c85 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -4,61 +4,24 @@ #[macro_use] extern crate lazy_static; -pub mod account_info; -pub mod account_overrides; -pub mod account_rent_state; -pub mod account_storage; -pub mod accounts; pub mod accounts_background_service; -pub mod accounts_cache; -pub mod accounts_db; -pub mod accounts_file; -pub mod accounts_hash; -pub mod accounts_index; -pub mod accounts_index_storage; -mod accounts_partition; -pub mod accounts_update_notifier_interface; -mod active_stats; -pub mod ancestors; -mod ancient_append_vecs; -pub mod append_vec; pub mod bank; pub mod bank_client; pub mod bank_forks; pub mod bank_utils; -pub mod blockhash_queue; -pub mod bucket_map_holder; -pub mod bucket_map_holder_stats; pub mod builtins; -pub mod cache_hash_data; -pub mod cache_hash_data_stats; pub mod commitment; -pub mod contains; -pub mod epoch_accounts_hash; mod epoch_rewards_hasher; pub mod epoch_stakes; pub mod genesis_utils; -pub mod hardened_unpack; -pub mod in_mem_accounts_index; pub mod inline_spl_associated_token_account; -pub mod inline_spl_token; -pub mod inline_spl_token_2022; pub mod loader_utils; pub mod non_circulating_supply; -pub mod nonce_info; -pub mod partitioned_rewards; pub mod prioritization_fee; pub mod prioritization_fee_cache; -mod pubkey_bins; -mod read_only_accounts_cache; -pub mod rent_collector; -pub mod rent_debits; -mod rolling_bit_field; pub mod root_bank_cache; pub mod runtime_config; -pub mod secondary_index; pub mod serde_snapshot; -mod shared_buffer_reader; pub mod snapshot_archive_info; pub mod snapshot_bank_utils; pub mod snapshot_config; @@ -66,26 +29,18 @@ pub mod snapshot_hash; pub mod snapshot_minimizer; pub mod snapshot_package; pub mod snapshot_utils; -pub mod sorted_storages; mod stake_account; pub mod stake_history; -mod stake_rewards; pub mod stake_weighted_timestamp; pub mod stakes; pub mod static_ids; pub mod status_cache; -mod storable_accounts; -pub mod tiered_storage; pub mod transaction_batch; -pub mod transaction_error_metrics; pub mod transaction_priority_details; -pub mod transaction_results; -mod verify_accounts_hash_in_background; pub mod vote_account; pub mod vote_parser; pub mod vote_sender_types; pub mod vote_transaction; -pub mod waitable_condvar; #[macro_use] extern crate solana_metrics; diff --git a/runtime/src/non_circulating_supply.rs b/runtime/src/non_circulating_supply.rs index 566160ad28..1c9f8dd9a4 100644 --- a/runtime/src/non_circulating_supply.rs +++ b/runtime/src/non_circulating_supply.rs @@ -1,9 +1,7 @@ use { - crate::{ - accounts_index::{AccountIndex, IndexKey, ScanConfig, ScanResult}, - bank::Bank, - }, + crate::bank::Bank, log::*, + solana_accounts_db::accounts_index::{AccountIndex, IndexKey, ScanConfig, ScanResult}, solana_sdk::{ account::ReadableAccount, pubkey::Pubkey, diff --git a/runtime/src/serde_snapshot.rs b/runtime/src/serde_snapshot.rs index c465322ccc..c929ae2f14 100644 --- a/runtime/src/serde_snapshot.rs +++ b/runtime/src/serde_snapshot.rs @@ -1,21 +1,8 @@ use { crate::{ - account_storage::meta::StoredMetaWriteVersion, - accounts::Accounts, - accounts_db::{ - AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId, - AtomicAppendVecId, BankHashStats, IndexGenerationInfo, - }, - accounts_file::AccountsFile, - accounts_hash::AccountsHash, - accounts_index::AccountSecondaryIndexes, - accounts_update_notifier_interface::AccountsUpdateNotifier, bank::{Bank, BankFieldsToDeserialize, BankRc}, - blockhash_queue::BlockhashQueue, builtins::BuiltinPrototype, - epoch_accounts_hash::EpochAccountsHash, epoch_stakes::EpochStakes, - rent_collector::RentCollector, runtime_config::RuntimeConfig, serde_snapshot::storage::SerializableAccountStorageEntry, snapshot_utils::{ @@ -26,6 +13,21 @@ use { bincode::{self, config::Options, Error}, log::*, serde::{de::DeserializeOwned, Deserialize, Serialize}, + solana_accounts_db::{ + account_storage::meta::StoredMetaWriteVersion, + accounts::Accounts, + accounts_db::{ + AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AccountsDbConfig, AppendVecId, + AtomicAppendVecId, BankHashStats, IndexGenerationInfo, + }, + accounts_file::AccountsFile, + accounts_hash::AccountsHash, + accounts_index::AccountSecondaryIndexes, + accounts_update_notifier_interface::AccountsUpdateNotifier, + blockhash_queue::BlockhashQueue, + epoch_accounts_hash::EpochAccountsHash, + rent_collector::RentCollector, + }, solana_measure::measure::Measure, solana_sdk::{ clock::{Epoch, Slot, UnixTimestamp}, @@ -55,12 +57,13 @@ use { mod newer; mod storage; mod tests; -mod types; mod utils; pub(crate) use { + solana_accounts_db::accounts_hash::{ + SerdeAccountsDeltaHash, SerdeAccountsHash, SerdeIncrementalAccountsHash, + }, storage::SerializedAppendVecId, - types::{SerdeAccountsDeltaHash, SerdeAccountsHash, SerdeIncrementalAccountsHash}, }; #[derive(Copy, Clone, Eq, PartialEq)] diff --git a/runtime/src/serde_snapshot/newer.rs b/runtime/src/serde_snapshot/newer.rs index e386120b1e..004e6e61d5 100644 --- a/runtime/src/serde_snapshot/newer.rs +++ b/runtime/src/serde_snapshot/newer.rs @@ -5,11 +5,10 @@ use { *, }, crate::{ - accounts_hash::AccountsHash, - ancestors::AncestorsForSerialization, bank::EpochRewardStatus, stakes::{serde_stakes_enum_compat, StakesEnum}, }, + solana_accounts_db::{accounts_hash::AccountsHash, ancestors::AncestorsForSerialization}, solana_measure::measure::Measure, solana_sdk::{deserialize_utils::ignore_eof_error, stake::state::Delegation}, std::{cell::RefCell, collections::HashSet, sync::RwLock}, diff --git a/runtime/src/serde_snapshot/storage.rs b/runtime/src/serde_snapshot/storage.rs index a1d8d15e70..1f9beab646 100644 --- a/runtime/src/serde_snapshot/storage.rs +++ b/runtime/src/serde_snapshot/storage.rs @@ -1,6 +1,6 @@ use { - crate::accounts_db::AccountStorageEntry, serde::{Deserialize, Serialize}, + solana_accounts_db::accounts_db::AccountStorageEntry, }; /// The serialized AppendVecId type is fixed as usize diff --git a/runtime/src/serde_snapshot/tests.rs b/runtime/src/serde_snapshot/tests.rs index f5ee7969c6..6724ab19d1 100644 --- a/runtime/src/serde_snapshot/tests.rs +++ b/runtime/src/serde_snapshot/tests.rs @@ -2,23 +2,6 @@ mod serde_snapshot_tests { use { crate::{ - account_storage::{AccountStorageMap, AccountStorageReference}, - accounts::Accounts, - accounts_db::{ - get_temp_accounts_paths, - test_utils::create_test_accounts, - tests::{ - assert_load_account, assert_not_load_account, check_accounts, check_storage, - create_account, linear_ancestors, modify_accounts, - }, - AccountShrinkThreshold, AccountStorageEntry, AccountsDb, AtomicAppendVecId, - VerifyAccountsHashAndLamportsConfig, - }, - accounts_file::{AccountsFile, AccountsFileError}, - accounts_hash::AccountsHash, - accounts_index::AccountSecondaryIndexes, - ancestors::Ancestors, - rent_collector::RentCollector, serde_snapshot::{ newer, reconstruct_accountsdb_from_fields, SerdeStyle, SerializableAccountsDb, SnapshotAccountsDbFields, TypeContext, @@ -28,6 +11,20 @@ mod serde_snapshot_tests { bincode::{serialize_into, Error}, log::info, rand::{thread_rng, Rng}, + solana_accounts_db::{ + account_storage::{AccountStorageMap, AccountStorageReference}, + accounts::Accounts, + accounts_db::{ + get_temp_accounts_paths, test_utils::create_test_accounts, AccountShrinkThreshold, + AccountStorageEntry, AccountsDb, AtomicAppendVecId, + VerifyAccountsHashAndLamportsConfig, + }, + accounts_file::{AccountsFile, AccountsFileError}, + accounts_hash::AccountsHash, + accounts_index::AccountSecondaryIndexes, + ancestors::Ancestors, + rent_collector::RentCollector, + }, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, clock::Slot, @@ -45,6 +42,14 @@ mod serde_snapshot_tests { tempfile::TempDir, }; + fn linear_ancestors(end_slot: u64) -> Ancestors { + let mut ancestors: Ancestors = vec![(0, 0)].into_iter().collect(); + for i in 1..end_slot { + ancestors.insert(i, (i - 1) as usize); + } + ancestors + } + fn context_accountsdb_from_stream<'a, C, R>( stream: &mut BufReader, account_paths: &[PathBuf], @@ -72,7 +77,7 @@ mod serde_snapshot_tests { None, AccountShrinkThreshold::default(), false, - Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), + Some(solana_accounts_db::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING), None, Arc::default(), None, @@ -302,7 +307,7 @@ mod serde_snapshot_tests { let db = reconstruct_accounts_db_via_serialization(&db, new_root); // Check root account exists - assert_load_account(&db, new_root, key2, 1); + db.assert_load_account(new_root, key2, 1); // Check purged account stays gone let unrooted_slot_ancestors = vec![(unrooted_slot, 1)].into_iter().collect(); @@ -319,21 +324,21 @@ mod serde_snapshot_tests { let mut pubkeys: Vec = vec![]; // Create 100 accounts in slot 0 - crate::accounts_db::tests::create_account(&accounts, &mut pubkeys, 0, 100, 0, 0); + accounts.create_account(&mut pubkeys, 0, 100, 0, 0); if pass == 0 { accounts.add_root_and_flush_write_cache(0); - check_storage(&accounts, 0, 100); + accounts.check_storage(0, 100); accounts.clean_accounts_for_tests(); - check_accounts(&accounts, &pubkeys, 0, 100, 1); + accounts.check_accounts(&pubkeys, 0, 100, 1); // clean should have done nothing continue; } // do some updates to those accounts and re-check - modify_accounts(&accounts, &pubkeys, 0, 100, 2); + accounts.modify_accounts(&pubkeys, 0, 100, 2); accounts.add_root_and_flush_write_cache(0); - check_storage(&accounts, 0, 100); - check_accounts(&accounts, &pubkeys, 0, 100, 2); + accounts.check_storage(0, 100); + accounts.check_accounts(&pubkeys, 0, 100, 2); accounts.calculate_accounts_delta_hash(0); let mut pubkeys1: Vec = vec![]; @@ -342,7 +347,7 @@ mod serde_snapshot_tests { let latest_slot = 1; // Modify the first 10 of the accounts from slot 0 in slot 1 - modify_accounts(&accounts, &pubkeys, latest_slot, 10, 3); + accounts.modify_accounts(&pubkeys, latest_slot, 10, 3); // Overwrite account 30 from slot 0 with lamports=0 into slot 1. // Slot 1 should now have 10 + 1 = 11 accounts let account = AccountSharedData::new(0, 0, AccountSharedData::default().owner()); @@ -350,18 +355,18 @@ mod serde_snapshot_tests { // Create 10 new accounts in slot 1, should now have 11 + 10 = 21 // accounts - create_account(&accounts, &mut pubkeys1, latest_slot, 10, 0, 0); + accounts.create_account(&mut pubkeys1, latest_slot, 10, 0, 0); accounts.calculate_accounts_delta_hash(latest_slot); accounts.add_root_and_flush_write_cache(latest_slot); - check_storage(&accounts, 1, 21); + accounts.check_storage(1, 21); // CREATE SLOT 2 let latest_slot = 2; let mut pubkeys2: Vec = vec![]; // Modify first 20 of the accounts from slot 0 in slot 2 - modify_accounts(&accounts, &pubkeys, latest_slot, 20, 4); + accounts.modify_accounts(&pubkeys, latest_slot, 20, 4); accounts.clean_accounts_for_tests(); // Overwrite account 31 from slot 0 with lamports=0 into slot 2. // Slot 2 should now have 20 + 1 = 21 accounts @@ -370,11 +375,11 @@ mod serde_snapshot_tests { // Create 10 new accounts in slot 2. Slot 2 should now have // 21 + 10 = 31 accounts - create_account(&accounts, &mut pubkeys2, latest_slot, 10, 0, 0); + accounts.create_account(&mut pubkeys2, latest_slot, 10, 0, 0); accounts.calculate_accounts_delta_hash(latest_slot); accounts.add_root_and_flush_write_cache(latest_slot); - check_storage(&accounts, 2, 31); + accounts.check_storage(2, 31); let ancestors = linear_ancestors(latest_slot); accounts.update_accounts_hash_for_tests(latest_slot, &ancestors, false, false); @@ -383,11 +388,11 @@ mod serde_snapshot_tests { // The first 20 accounts of slot 0 have been updated in slot 2, as well as // accounts 30 and 31 (overwritten with zero-lamport accounts in slot 1 and // slot 2 respectively), so only 78 accounts are left in slot 0's storage entries. - check_storage(&accounts, 0, 78); + accounts.check_storage(0, 78); // 10 of the 21 accounts have been modified in slot 2, so only 11 // accounts left in slot 1. - check_storage(&accounts, 1, 11); - check_storage(&accounts, 2, 31); + accounts.check_storage(1, 11); + accounts.check_storage(2, 31); let daccounts = reconstruct_accounts_db_via_serialization(&accounts, latest_slot); @@ -412,11 +417,11 @@ mod serde_snapshot_tests { daccounts.print_count_and_status("daccounts"); // Don't check the first 35 accounts which have not been modified on slot 0 - check_accounts(&daccounts, &pubkeys[35..], 0, 65, 37); - check_accounts(&daccounts, &pubkeys1, 1, 10, 1); - check_storage(&daccounts, 0, 100); - check_storage(&daccounts, 1, 21); - check_storage(&daccounts, 2, 31); + daccounts.check_accounts(&pubkeys[35..], 0, 65, 37); + daccounts.check_accounts(&pubkeys1, 1, 10, 1); + daccounts.check_storage(0, 100); + daccounts.check_storage(1, 21); + daccounts.check_storage(2, 31); assert_eq!( daccounts.update_accounts_hash_for_tests(latest_slot, &ancestors, false, false,), @@ -462,7 +467,7 @@ mod serde_snapshot_tests { } accounts.add_root_and_flush_write_cache(current_slot); - assert_load_account(&accounts, current_slot, pubkey, zero_lamport); + accounts.assert_load_account(current_slot, pubkey, zero_lamport); accounts.print_accounts_stats("accounts"); @@ -481,7 +486,7 @@ mod serde_snapshot_tests { accounts.print_accounts_stats("reconstructed"); - assert_load_account(&accounts, current_slot, pubkey, zero_lamport); + accounts.assert_load_account(current_slot, pubkey, zero_lamport); } fn with_chained_zero_lamport_accounts(f: F) @@ -534,10 +539,10 @@ mod serde_snapshot_tests { accounts.print_accounts_stats("post_f"); - assert_load_account(&accounts, current_slot, pubkey, some_lamport); - assert_load_account(&accounts, current_slot, purged_pubkey1, 0); - assert_load_account(&accounts, current_slot, purged_pubkey2, 0); - assert_load_account(&accounts, current_slot, dummy_pubkey, dummy_lamport); + accounts.assert_load_account(current_slot, pubkey, some_lamport); + accounts.assert_load_account(current_slot, purged_pubkey1, 0); + accounts.assert_load_account(current_slot, purged_pubkey2, 0); + accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport); let ancestors = Ancestors::default(); let epoch_schedule = EpochSchedule::default(); @@ -636,9 +641,9 @@ mod serde_snapshot_tests { accounts.clean_accounts_for_tests(); accounts.print_count_and_status("after purge zero"); - assert_load_account(&accounts, current_slot, pubkey, old_lamport); - assert_load_account(&accounts, current_slot, purged_pubkey1, 0); - assert_load_account(&accounts, current_slot, purged_pubkey2, 0); + accounts.assert_load_account(current_slot, pubkey, old_lamport); + accounts.assert_load_account(current_slot, purged_pubkey1, 0); + accounts.assert_load_account(current_slot, purged_pubkey2, 0); } #[test] @@ -731,9 +736,9 @@ mod serde_snapshot_tests { accounts.calculate_accounts_delta_hash(current_slot); accounts.add_root(current_slot); - assert_load_account(&accounts, current_slot, pubkey1, zero_lamport); - assert_load_account(&accounts, current_slot, pubkey2, old_lamport); - assert_load_account(&accounts, current_slot, dummy_pubkey, dummy_lamport); + accounts.assert_load_account(current_slot, pubkey1, zero_lamport); + accounts.assert_load_account(current_slot, pubkey2, old_lamport); + accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport); // At this point, there is no index entries for A and B // If step C and step D should be purged, snapshot restore would cause @@ -752,9 +757,9 @@ mod serde_snapshot_tests { info!("pubkey: {}", pubkey1); accounts.print_accounts_stats("pre_clean"); - assert_load_account(&accounts, current_slot, pubkey1, zero_lamport); - assert_load_account(&accounts, current_slot, pubkey2, old_lamport); - assert_load_account(&accounts, current_slot, dummy_pubkey, dummy_lamport); + accounts.assert_load_account(current_slot, pubkey1, zero_lamport); + accounts.assert_load_account(current_slot, pubkey2, old_lamport); + accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport); // F: Finally, make Step A cleanable current_slot += 1; @@ -770,9 +775,9 @@ mod serde_snapshot_tests { accounts.clean_accounts_for_tests(); // Ensure pubkey2 is cleaned from the index finally - assert_not_load_account(&accounts, current_slot, pubkey1); - assert_load_account(&accounts, current_slot, pubkey2, old_lamport); - assert_load_account(&accounts, current_slot, dummy_pubkey, dummy_lamport); + accounts.assert_not_load_account(current_slot, pubkey1); + accounts.assert_load_account(current_slot, pubkey2, old_lamport); + accounts.assert_load_account(current_slot, dummy_pubkey, dummy_lamport); } #[test] diff --git a/runtime/src/serde_snapshot/types.rs b/runtime/src/serde_snapshot/types.rs deleted file mode 100644 index ee739a3c61..0000000000 --- a/runtime/src/serde_snapshot/types.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Types used by snapshots for safe serialization/deserialization -use { - crate::accounts_hash::{AccountsDeltaHash, AccountsHash, IncrementalAccountsHash}, - serde::{Deserialize, Serialize}, - solana_sdk::hash::Hash, -}; - -/// Snapshot serde-safe accounts delta hash -#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, AbiExample)] -pub struct SerdeAccountsDeltaHash(pub Hash); - -impl From for AccountsDeltaHash { - fn from(accounts_delta_hash: SerdeAccountsDeltaHash) -> Self { - Self(accounts_delta_hash.0) - } -} -impl From for SerdeAccountsDeltaHash { - fn from(accounts_delta_hash: AccountsDeltaHash) -> Self { - Self(accounts_delta_hash.0) - } -} - -/// Snapshot serde-safe accounts hash -#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, AbiExample)] -pub struct SerdeAccountsHash(pub Hash); - -impl From for AccountsHash { - fn from(accounts_hash: SerdeAccountsHash) -> Self { - Self(accounts_hash.0) - } -} -impl From for SerdeAccountsHash { - fn from(accounts_hash: AccountsHash) -> Self { - Self(accounts_hash.0) - } -} - -/// Snapshot serde-safe incremental accounts hash -#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq, AbiExample)] -pub struct SerdeIncrementalAccountsHash(pub Hash); - -impl From for IncrementalAccountsHash { - fn from(incremental_accounts_hash: SerdeIncrementalAccountsHash) -> Self { - Self(incremental_accounts_hash.0) - } -} -impl From for SerdeIncrementalAccountsHash { - fn from(incremental_accounts_hash: IncrementalAccountsHash) -> Self { - Self(incremental_accounts_hash.0) - } -} diff --git a/runtime/src/snapshot_bank_utils.rs b/runtime/src/snapshot_bank_utils.rs index ecddfef0df..267f909fe5 100644 --- a/runtime/src/snapshot_bank_utils.rs +++ b/runtime/src/snapshot_bank_utils.rs @@ -1,12 +1,5 @@ use { crate::{ - accounts_db::{ - AccountShrinkThreshold, AccountStorageEntry, AccountsDbConfig, AtomicAppendVecId, - CalcAccountsHashDataSource, - }, - accounts_hash::AccountsHash, - accounts_index::AccountSecondaryIndexes, - accounts_update_notifier_interface::AccountsUpdateNotifier, bank::{Bank, BankFieldsToDeserialize, BankSlotDelta}, builtins::BuiltinPrototype, runtime_config::RuntimeConfig, @@ -36,6 +29,15 @@ use { bincode::{config::Options, serialize_into}, fs_err, log::*, + solana_accounts_db::{ + accounts_db::{ + AccountShrinkThreshold, AccountStorageEntry, AccountsDbConfig, AtomicAppendVecId, + CalcAccountsHashDataSource, + }, + accounts_hash::AccountsHash, + accounts_index::AccountSecondaryIndexes, + accounts_update_notifier_interface::AccountsUpdateNotifier, + }, solana_measure::{measure, measure::Measure}, solana_sdk::{ clock::Slot, @@ -1253,8 +1255,6 @@ mod tests { use { super::*, crate::{ - accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING, - accounts_hash::{CalcAccountsHashConfig, HashStats}, genesis_utils, snapshot_utils::{ clean_orphaned_account_snapshot_dirs, create_all_accounts_run_and_snapshot_dirs, @@ -1264,9 +1264,13 @@ mod tests { purge_old_bank_snapshots, purge_old_bank_snapshots_at_startup, snapshot_storage_rebuilder::get_slot_and_append_vec_id, ArchiveFormat, }, - sorted_storages::SortedStorages, status_cache::Status, }, + solana_accounts_db::{ + accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING, + accounts_hash::{CalcAccountsHashConfig, HashStats}, + sorted_storages::SortedStorages, + }, solana_sdk::{ genesis_config::create_genesis_config, native_token::{sol_to_lamports, LAMPORTS_PER_SOL}, diff --git a/runtime/src/snapshot_hash.rs b/runtime/src/snapshot_hash.rs index 814578c079..daa1538b07 100644 --- a/runtime/src/snapshot_hash.rs +++ b/runtime/src/snapshot_hash.rs @@ -1,6 +1,6 @@ //! Helper types and functions for handling and dealing with snapshot hashes. use { - crate::{accounts_hash::AccountsHashEnum, epoch_accounts_hash::EpochAccountsHash}, + solana_accounts_db::{accounts_hash::AccountsHashEnum, epoch_accounts_hash::EpochAccountsHash}, solana_sdk::{ clock::Slot, hash::{Hash, Hasher}, diff --git a/runtime/src/snapshot_minimizer.rs b/runtime/src/snapshot_minimizer.rs index b8d5a09754..dc1276b8de 100644 --- a/runtime/src/snapshot_minimizer.rs +++ b/runtime/src/snapshot_minimizer.rs @@ -1,21 +1,19 @@ //! Used to create minimal snapshots - separated here to keep accounts_db simpler use { - crate::{ - accounts_db::{ - AccountStorageEntry, AccountsDb, GetUniqueAccountsResult, PurgeStats, StoreReclaims, - }, - accounts_partition, - bank::Bank, - builtins::BUILTINS, - static_ids, - }, + crate::{bank::Bank, builtins::BUILTINS, static_ids}, dashmap::DashSet, log::info, rayon::{ iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}, prelude::ParallelSlice, }, + solana_accounts_db::{ + accounts_db::{ + AccountStorageEntry, AccountsDb, GetUniqueAccountsResult, PurgeStats, StoreReclaims, + }, + accounts_partition, + }, solana_measure::measure, solana_sdk::{ account::ReadableAccount, @@ -373,7 +371,7 @@ impl<'a> SnapshotMinimizer<'a> { ( slot, &accounts[..], - crate::accounts_db::INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION, + solana_accounts_db::accounts_db::INCLUDE_SLOT_IN_HASH_IRRELEVANT_APPEND_VEC_OPERATION, ), Some(hashes), new_storage, diff --git a/runtime/src/snapshot_package.rs b/runtime/src/snapshot_package.rs index dcac74e2ee..c9eef2ab1f 100644 --- a/runtime/src/snapshot_package.rs +++ b/runtime/src/snapshot_package.rs @@ -1,16 +1,18 @@ use { crate::{ - accounts::Accounts, - accounts_db::{AccountStorageEntry, IncludeSlotInHash, INCLUDE_SLOT_IN_HASH_TESTS}, - accounts_hash::{AccountsHash, AccountsHashEnum}, bank::Bank, - epoch_accounts_hash::EpochAccountsHash, - rent_collector::RentCollector, snapshot_archive_info::{SnapshotArchiveInfo, SnapshotArchiveInfoGetter}, snapshot_hash::SnapshotHash, snapshot_utils::{self, ArchiveFormat, BankSnapshotInfo, SnapshotVersion}, }, log::*, + solana_accounts_db::{ + accounts::Accounts, + accounts_db::{AccountStorageEntry, IncludeSlotInHash, INCLUDE_SLOT_IN_HASH_TESTS}, + accounts_hash::{AccountsHash, AccountsHashEnum}, + epoch_accounts_hash::EpochAccountsHash, + rent_collector::RentCollector, + }, solana_sdk::{clock::Slot, feature_set, sysvar::epoch_schedule::EpochSchedule}, std::{ path::{Path, PathBuf}, diff --git a/runtime/src/snapshot_utils.rs b/runtime/src/snapshot_utils.rs index ec33283cff..22738cfc89 100644 --- a/runtime/src/snapshot_utils.rs +++ b/runtime/src/snapshot_utils.rs @@ -1,15 +1,6 @@ use { crate::{ - account_storage::AccountStorageMap, - accounts_db::{AccountStorageEntry, AtomicAppendVecId}, - accounts_file::AccountsFileError, - append_vec::AppendVec, - hardened_unpack::{ - streaming_unpack_snapshot, unpack_snapshot, ParallelSelector, UnpackError, - UnpackedAppendVecMap, - }, serde_snapshot::SnapshotStreams, - shared_buffer_reader::{SharedBuffer, SharedBufferReader}, snapshot_archive_info::{ FullSnapshotArchiveInfo, IncrementalSnapshotArchiveInfo, SnapshotArchiveInfoGetter, }, @@ -27,6 +18,19 @@ use { log::*, rayon::prelude::*, regex::Regex, + solana_accounts_db::{ + account_storage::AccountStorageMap, + accounts_db::{ + self, create_accounts_run_and_snapshot_dirs, AccountStorageEntry, AtomicAppendVecId, + }, + accounts_file::AccountsFileError, + append_vec::AppendVec, + hardened_unpack::{ + streaming_unpack_snapshot, unpack_snapshot, ParallelSelector, UnpackError, + UnpackedAppendVecMap, + }, + shared_buffer_reader::{SharedBuffer, SharedBufferReader}, + }, solana_measure::{measure, measure::Measure}, solana_sdk::{clock::Slot, hash::Hash}, std::{ @@ -38,7 +42,7 @@ use { path::{Path, PathBuf}, process::ExitStatus, str::FromStr, - sync::{atomic::AtomicU32, Arc}, + sync::{atomic::AtomicU32, Arc, Mutex}, thread::{Builder, JoinHandle}, }, tar::{self, Archive}, @@ -49,7 +53,6 @@ use { mod archive_format; pub mod snapshot_storage_rebuilder; pub use archive_format::*; -use std::sync::Mutex; pub const SNAPSHOT_STATUS_CACHE_FILENAME: &str = "status_cache"; pub const SNAPSHOT_VERSION_FILENAME: &str = "version"; @@ -482,25 +485,8 @@ pub fn create_and_canonicalize_directories(directories: &[PathBuf]) -> Result) { - match fs_err::read_dir(path.as_ref()) { - Err(err) => { - warn!("Failed to delete contents: {err}") - } - Ok(dir_entries) => { - for entry in dir_entries.flatten() { - let sub_path = entry.path(); - let result = if sub_path.is_dir() { - fs_err::remove_dir_all(&sub_path) - } else { - fs_err::remove_file(&sub_path) - }; - if let Err(err) = result { - warn!("Failed to delete contents: {err}"); - } - } - } - } +pub(crate) fn delete_contents_of_path(path: impl AsRef) { + accounts_db::delete_contents_of_path(path) } /// Moves and asynchronously deletes the contents of a directory to avoid blocking on it. @@ -753,7 +739,7 @@ pub fn archive_snapshot_package( for storage in snapshot_package.snapshot_storages.iter() { storage.flush()?; let storage_path = storage.get_path(); - let output_path = staging_accounts_dir.join(crate::append_vec::AppendVec::file_name( + let output_path = staging_accounts_dir.join(AppendVec::file_name( storage.slot(), storage.append_vec_id(), )); @@ -1116,55 +1102,15 @@ fn check_deserialize_file_consumed( Ok(()) } -/// To allow generating a bank snapshot directory with full state information, we need to -/// hardlink account appendvec files from the runtime operation directory to a snapshot -/// hardlink directory. This is to create the run/ and snapshot sub directories for an -/// account_path provided by the user. These two sub directories are on the same file -/// system partition to allow hard-linking. -pub fn create_accounts_run_and_snapshot_dirs( - account_dir: impl AsRef, -) -> std::io::Result<(PathBuf, PathBuf)> { - let run_path = account_dir.as_ref().join("run"); - let snapshot_path = account_dir.as_ref().join("snapshot"); - if (!run_path.is_dir()) || (!snapshot_path.is_dir()) { - // If the "run/" or "snapshot" sub directories do not exist, the directory may be from - // an older version for which the appendvec files are at this directory. Clean up - // them first. - // This will be done only once when transitioning from an old image without run directory - // to this new version using run and snapshot directories. - // The run/ content cleanup will be done at a later point. The snapshot/ content persists - // across the process boot, and will be purged by the account_background_service. - if fs_err::remove_dir_all(&account_dir).is_err() { - delete_contents_of_path(&account_dir); - } - fs_err::create_dir_all(&run_path)?; - fs_err::create_dir_all(&snapshot_path)?; - } - - Ok((run_path, snapshot_path)) -} - /// For all account_paths, create the run/ and snapshot/ sub directories. /// If an account_path directory does not exist, create it. /// It returns (account_run_paths, account_snapshot_paths) or error pub fn create_all_accounts_run_and_snapshot_dirs( account_paths: &[PathBuf], ) -> Result<(Vec, Vec)> { - let mut run_dirs = Vec::with_capacity(account_paths.len()); - let mut snapshot_dirs = Vec::with_capacity(account_paths.len()); - for account_path in account_paths { - // create the run/ and snapshot/ sub directories for each account_path - let (run_dir, snapshot_dir) = - create_accounts_run_and_snapshot_dirs(account_path).map_err(|err| { - SnapshotError::IoWithSource( - err, - "Unable to create account run and snapshot directories", - ) - })?; - run_dirs.push(run_dir); - snapshot_dirs.push(snapshot_dir); - } - Ok((run_dirs, snapshot_dirs)) + accounts_db::create_all_accounts_run_and_snapshot_dirs(account_paths).map_err(|err| { + SnapshotError::IoWithSource(err, "Unable to create account run and snapshot directories") + }) } /// Return account path from the appendvec path after checking its format. diff --git a/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs b/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs index 63b25bc523..4971c694ba 100644 --- a/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs +++ b/runtime/src/snapshot_utils/snapshot_storage_rebuilder.rs @@ -4,14 +4,9 @@ use { super::{ get_io_error, snapshot_version_from_file, SnapshotError, SnapshotFrom, SnapshotVersion, }, - crate::{ - account_storage::{AccountStorageMap, AccountStorageReference}, - accounts_db::{AccountStorageEntry, AccountsDb, AppendVecId, AtomicAppendVecId}, - append_vec::AppendVec, - serde_snapshot::{ - self, reconstruct_single_storage, remap_and_reconstruct_single_storage, - snapshot_storage_lengths_from_fields, SerdeStyle, SerializedAppendVecId, - }, + crate::serde_snapshot::{ + self, reconstruct_single_storage, remap_and_reconstruct_single_storage, + snapshot_storage_lengths_from_fields, SerdeStyle, SerializedAppendVecId, }, crossbeam_channel::{select, unbounded, Receiver, Sender}, dashmap::DashMap, @@ -21,6 +16,11 @@ use { ThreadPool, ThreadPoolBuilder, }, regex::Regex, + solana_accounts_db::{ + account_storage::{AccountStorageMap, AccountStorageReference}, + accounts_db::{AccountStorageEntry, AccountsDb, AppendVecId, AtomicAppendVecId}, + append_vec::AppendVec, + }, solana_sdk::clock::Slot, std::{ collections::HashMap, @@ -464,8 +464,8 @@ pub(crate) fn get_slot_and_append_vec_id(filename: &str) -> (Slot, usize) { #[cfg(test)] mod tests { use { - super::*, - crate::{append_vec::AppendVec, snapshot_utils::SNAPSHOT_VERSION_FILENAME}, + super::*, crate::snapshot_utils::SNAPSHOT_VERSION_FILENAME, + solana_accounts_db::append_vec::AppendVec, }; #[test] diff --git a/runtime/src/stake_rewards.rs b/runtime/src/stake_rewards.rs deleted file mode 100644 index 55ab5844dc..0000000000 --- a/runtime/src/stake_rewards.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Code for stake and vote rewards - -use { - crate::bank::RewardInfo, - solana_sdk::{account::AccountSharedData, pubkey::Pubkey}, -}; - -#[derive(AbiExample, Debug, Serialize, Deserialize, Clone, PartialEq)] -pub(crate) struct StakeReward { - pub(crate) stake_pubkey: Pubkey, - pub(crate) stake_reward_info: RewardInfo, - pub(crate) stake_account: AccountSharedData, -} - -impl StakeReward { - pub(crate) fn get_stake_reward(&self) -> i64 { - self.stake_reward_info.lamports - } -} diff --git a/runtime/src/stakes.rs b/runtime/src/stakes.rs index 91b308d762..243e350213 100644 --- a/runtime/src/stakes.rs +++ b/runtime/src/stakes.rs @@ -4,7 +4,6 @@ use { crate::{ stake_account, stake_history::StakeHistory, - stake_rewards::StakeReward, vote_account::{VoteAccount, VoteAccounts}, }, dashmap::DashMap, @@ -13,6 +12,7 @@ use { num_derive::ToPrimitive, num_traits::ToPrimitive, rayon::{prelude::*, ThreadPool}, + solana_accounts_db::stake_rewards::StakeReward, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, clock::{Epoch, Slot}, diff --git a/runtime/src/static_ids.rs b/runtime/src/static_ids.rs index 5d9909e89d..4cedf3b847 100644 --- a/runtime/src/static_ids.rs +++ b/runtime/src/static_ids.rs @@ -1,5 +1,6 @@ use { - crate::{inline_spl_associated_token_account, inline_spl_token, inline_spl_token_2022}, + crate::inline_spl_associated_token_account, + solana_accounts_db::{inline_spl_token, inline_spl_token_2022}, solana_sdk::pubkey::Pubkey, }; diff --git a/runtime/src/status_cache.rs b/runtime/src/status_cache.rs index 61e66f2500..f15b5556c2 100644 --- a/runtime/src/status_cache.rs +++ b/runtime/src/status_cache.rs @@ -1,8 +1,8 @@ use { - crate::ancestors::Ancestors, log::*, rand::{thread_rng, Rng}, serde::Serialize, + solana_accounts_db::ancestors::Ancestors, solana_sdk::{ clock::{Slot, MAX_RECENT_BLOCKHASHES}, hash::Hash, diff --git a/runtime/store-tool/Cargo.toml b/runtime/store-tool/Cargo.toml index d3252e3a12..2187a5f77b 100644 --- a/runtime/store-tool/Cargo.toml +++ b/runtime/store-tool/Cargo.toml @@ -12,8 +12,8 @@ edition = { workspace = true } [dependencies] clap = { workspace = true } log = { workspace = true } +solana-accounts-db = { workspace = true } solana-logger = { workspace = true } -solana-runtime = { workspace = true } solana-sdk = { workspace = true } solana-version = { workspace = true } diff --git a/runtime/store-tool/src/main.rs b/runtime/store-tool/src/main.rs index e242a73792..98140ed59b 100644 --- a/runtime/store-tool/src/main.rs +++ b/runtime/store-tool/src/main.rs @@ -1,7 +1,7 @@ use { clap::{crate_description, crate_name, value_t, value_t_or_exit, App, Arg}, log::*, - solana_runtime::{account_storage::meta::StoredAccountMeta, append_vec::AppendVec}, + solana_accounts_db::{account_storage::meta::StoredAccountMeta, append_vec::AppendVec}, solana_sdk::{ account::{AccountSharedData, ReadableAccount}, hash::Hash, diff --git a/runtime/tests/accounts.rs b/runtime/tests/accounts.rs index 98aebe8ab8..8e71f26485 100644 --- a/runtime/tests/accounts.rs +++ b/runtime/tests/accounts.rs @@ -2,7 +2,7 @@ use { log::*, rand::{thread_rng, Rng}, rayon::prelude::*, - solana_runtime::{ + solana_accounts_db::{ accounts_db::{AccountsDb, LoadHint, INCLUDE_SLOT_IN_HASH_TESTS}, ancestors::Ancestors, }, diff --git a/runtime/tests/stake.rs b/runtime/tests/stake.rs index 4141fa4d9f..7ac763a50a 100755 --- a/runtime/tests/stake.rs +++ b/runtime/tests/stake.rs @@ -1,10 +1,10 @@ #![allow(clippy::integer_arithmetic)] use { + solana_accounts_db::epoch_accounts_hash::EpochAccountsHash, solana_runtime::{ bank::Bank, bank_client::BankClient, - epoch_accounts_hash::EpochAccountsHash, genesis_utils::{create_genesis_config_with_leader, GenesisConfigInfo}, }, solana_sdk::{ diff --git a/test-validator/Cargo.toml b/test-validator/Cargo.toml index b329150069..60f299d01e 100644 --- a/test-validator/Cargo.toml +++ b/test-validator/Cargo.toml @@ -16,6 +16,7 @@ crossbeam-channel = { workspace = true } log = { workspace = true } serde_derive = { workspace = true } serde_json = { workspace = true } +solana-accounts-db = { workspace = true } solana-cli-output = { workspace = true } solana-client = { workspace = true } solana-core = { workspace = true } diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index 56c3d5bd02..f657aaee1c 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -3,6 +3,11 @@ use { base64::{prelude::BASE64_STANDARD, Engine}, crossbeam_channel::Receiver, log::*, + solana_accounts_db::{ + accounts_db::{create_accounts_run_and_snapshot_dirs, AccountsDbConfig}, + accounts_index::AccountsIndexConfig, + hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + }, solana_cli_output::CliAccount, solana_client::rpc_request::MAX_MULTIPLE_ACCOUNTS, solana_core::{ @@ -28,10 +33,8 @@ use { solana_rpc::{rpc::JsonRpcConfig, rpc_pubsub_service::PubSubConfig}, solana_rpc_client::{nonblocking, rpc_client::RpcClient}, solana_runtime::{ - accounts_db::AccountsDbConfig, accounts_index::AccountsIndexConfig, bank_forks::BankForks, - genesis_utils::create_genesis_config_with_leader_ex, - hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, runtime_config::RuntimeConfig, - snapshot_config::SnapshotConfig, snapshot_utils::create_accounts_run_and_snapshot_dirs, + bank_forks::BankForks, genesis_utils::create_genesis_config_with_leader_ex, + runtime_config::RuntimeConfig, snapshot_config::SnapshotConfig, }, solana_sdk::{ account::{Account, AccountSharedData}, diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 03d14f17d4..845bdda7ee 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -33,6 +33,7 @@ rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serde_yaml = { workspace = true } +solana-accounts-db = { workspace = true } solana-clap-utils = { workspace = true } solana-cli-config = { workspace = true } solana-core = { workspace = true } diff --git a/validator/src/admin_rpc_service.rs b/validator/src/admin_rpc_service.rs index 3cf1878dbb..8cbaf1e5f7 100644 --- a/validator/src/admin_rpc_service.rs +++ b/validator/src/admin_rpc_service.rs @@ -9,6 +9,7 @@ use { jsonrpc_server_utils::tokio, log::*, serde::{de::Deserializer, Deserialize, Serialize}, + solana_accounts_db::accounts_index::AccountIndex, solana_core::{ admin_rpc_post_init::AdminRpcRequestMetadataPostInit, consensus::{tower_storage::TowerStorage, Tower}, @@ -18,7 +19,6 @@ use { solana_gossip::contact_info::{ContactInfo, Protocol, SOCKET_ADDR_UNSPECIFIED}, solana_rpc::rpc::verify_pubkey, solana_rpc_client_api::{config::RpcAccountIndex, custom_error::RpcCustomError}, - solana_runtime::accounts_index::AccountIndex, solana_sdk::{ exit::Exit, pubkey::Pubkey, @@ -865,16 +865,17 @@ mod tests { super::*, rand::{distributions::Uniform, thread_rng, Rng}, serde_json::Value, + solana_accounts_db::{ + accounts_index::AccountSecondaryIndexes, inline_spl_token, + secondary_index::MAX_NUM_LARGEST_INDEX_KEYS_RETURNED, + }, solana_core::consensus::tower_storage::NullTowerStorage, solana_gossip::cluster_info::ClusterInfo, solana_ledger::genesis_utils::{create_genesis_config, GenesisConfigInfo}, solana_rpc::rpc::create_validator_exit, solana_runtime::{ - accounts_index::AccountSecondaryIndexes, bank::{Bank, BankTestConfig}, bank_forks::BankForks, - inline_spl_token, - secondary_index::MAX_NUM_LARGEST_INDEX_KEYS_RETURNED, }, solana_sdk::{ account::{Account, AccountSharedData}, diff --git a/validator/src/bin/solana-test-validator.rs b/validator/src/bin/solana-test-validator.rs index 7ebc2bbfbf..69f749cfcb 100644 --- a/validator/src/bin/solana-test-validator.rs +++ b/validator/src/bin/solana-test-validator.rs @@ -3,6 +3,7 @@ use { crossbeam_channel::unbounded, itertools::Itertools, log::*, + solana_accounts_db::accounts_index::{AccountIndex, AccountSecondaryIndexes}, solana_clap_utils::{ input_parsers::{pubkey_of, pubkeys_of, value_of}, input_validators::normalize_to_url_if_moniker, @@ -14,7 +15,6 @@ use { rpc_pubsub_service::PubSubConfig, }, solana_rpc_client::rpc_client::RpcClient, - solana_runtime::accounts_index::{AccountIndex, AccountSecondaryIndexes}, solana_sdk::{ account::AccountSharedData, clock::Slot, diff --git a/validator/src/cli.rs b/validator/src/cli.rs index 9383925541..8126150c57 100644 --- a/validator/src/cli.rs +++ b/validator/src/cli.rs @@ -3,6 +3,12 @@ use { crate_description, crate_name, App, AppSettings, Arg, ArgGroup, ArgMatches, SubCommand, }, log::warn, + solana_accounts_db::{ + accounts_db::{ + DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE, DEFAULT_ACCOUNTS_SHRINK_RATIO, + }, + hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, + }, solana_clap_utils::{ hidden_unless_forced, input_validators::{ @@ -23,10 +29,6 @@ use { solana_rpc::{rpc::MAX_REQUEST_BODY_SIZE, rpc_pubsub_service::PubSubConfig}, solana_rpc_client_api::request::MAX_MULTIPLE_ACCOUNTS, solana_runtime::{ - accounts_db::{ - DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE, DEFAULT_ACCOUNTS_SHRINK_RATIO, - }, - hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, snapshot_bank_utils::{ DEFAULT_FULL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, DEFAULT_INCREMENTAL_SNAPSHOT_ARCHIVE_INTERVAL_SLOTS, diff --git a/validator/src/main.rs b/validator/src/main.rs index 47e2f57adf..3ab70fae19 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -7,6 +7,17 @@ use { crossbeam_channel::unbounded, log::*, rand::{seq::SliceRandom, thread_rng}, + solana_accounts_db::{ + accounts_db::{ + AccountShrinkThreshold, AccountsDb, AccountsDbConfig, CreateAncientStorage, + FillerAccountsConfig, + }, + accounts_index::{ + AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude, + AccountsIndexConfig, IndexLimitMb, + }, + partitioned_rewards::TestPartitionedEpochRewards, + }, solana_clap_utils::input_parsers::{keypair_of, keypairs_of, pubkey_of, value_of}, solana_core::{ banking_trace::DISABLED_BAKING_TRACE_DIR, @@ -36,15 +47,6 @@ use { solana_rpc_client::rpc_client::RpcClient, solana_rpc_client_api::config::RpcLeaderScheduleConfig, solana_runtime::{ - accounts_db::{ - AccountShrinkThreshold, AccountsDb, AccountsDbConfig, CreateAncientStorage, - FillerAccountsConfig, - }, - accounts_index::{ - AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude, - AccountsIndexConfig, IndexLimitMb, - }, - partitioned_rewards::TestPartitionedEpochRewards, runtime_config::RuntimeConfig, snapshot_bank_utils::DISABLED_SNAPSHOT_ARCHIVE_INTERVAL, snapshot_config::{SnapshotConfig, SnapshotUsage},