blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add accounts-filler-size command line option (#23896)

This commit is contained in:
Giorgio Gambino 2022-04-11 19:10:09 +01:00 committed by GitHub
parent eb478d72d1
commit 60b2155bd3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 83 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/src/validator.rs
View File

@ -798,8 +798,7 @@ impl Validator {
let enable_gossip_push = config
.accounts_db_config
.as_ref()
.and_then(|config| config.filler_account_count)
.map(|count| count == 0)
.map(|config| config.filler_accounts_config.count == 0)
.unwrap_or(true);
let snapshot_packager_service = SnapshotPackagerService::new(

20
ledger-tool/src/main.rs
View File

@ -32,7 +32,7 @@ use {
},
solana_measure::measure::Measure,
solana_runtime::{
accounts_db::AccountsDbConfig,
accounts_db::{AccountsDbConfig, FillerAccountsConfig},
accounts_index::{AccountsIndexConfig, ScanConfig},
bank::{Bank, RewardCalculationEvent},
bank_forks::BankForks,
@ -928,7 +928,16 @@ fn main() {
.value_name("COUNT")
.validator(is_parsable::<usize>)
.takes_value(true)
.default_value("0")
.help("How many accounts to add to stress the system. Accounts are ignored in operations related to correctness.");
let accounts_filler_size = Arg::with_name("accounts_filler_size")
.long("accounts-filler-size")
.value_name("BYTES")
.validator(is_parsable::<usize>)
.takes_value(true)
.default_value("0")
.requires("accounts_filler_count")
.help("Size per filler account in bytes.");
let account_paths_arg = Arg::with_name("account_paths")
.long("accounts")
.value_name("PATHS")
@ -1283,6 +1292,7 @@ fn main() {
.arg(&disable_disk_index)
.arg(&accountsdb_skip_shrink)
.arg(&accounts_filler_count)
.arg(&accounts_filler_size)
.arg(&verify_index_arg)
.arg(&hard_forks_arg)
.arg(&no_accounts_db_caching_arg)
@ -2066,13 +2076,15 @@ fn main() {
accounts_index_config.drives = Some(accounts_index_paths);
}
let filler_account_count =
value_t!(arg_matches, "accounts_filler_count", usize).ok();
let filler_accounts_config = FillerAccountsConfig {
count: value_t_or_exit!(arg_matches, "accounts_filler_count", usize),
size: value_t_or_exit!(arg_matches, "accounts_filler_size", usize),
};
let accounts_db_config = Some(AccountsDbConfig {
index: Some(accounts_index_config),
accounts_hash_cache_path: Some(ledger_path.clone()),
filler_account_count,
filler_accounts_config,
..AccountsDbConfig::default()
});

9
ledger/src/bank_forks_utils.rs
View File

@ -125,13 +125,12 @@ pub fn load_bank_forks(
accounts_update_notifier,
)
} else {
if process_options
let maybe_filler_accounts = process_options
.accounts_db_config
.as_ref()
.and_then(|config| config.filler_account_count)
.unwrap_or_default()
> 0
{
.map(|config| config.filler_accounts_config.count > 0);
if let Some(true) = maybe_filler_accounts {
panic!("filler accounts specified, but not loading from snapshot");
}

61
runtime/src/accounts_db.rs
View File

@ -65,6 +65,7 @@ use {
genesis_config::{ClusterType, GenesisConfig},
hash::Hash,
pubkey::Pubkey,
rent::Rent,
timing::AtomicInterval,
},
std::{
@ -128,14 +129,14 @@ const CACHE_VIRTUAL_STORED_SIZE: StoredSize = 0;
pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig {
index: Some(ACCOUNTS_INDEX_CONFIG_FOR_TESTING),
accounts_hash_cache_path: None,
filler_account_count: None,
filler_accounts_config: FillerAccountsConfig::const_default(),
hash_calc_num_passes: None,
write_cache_limit_bytes: None,
};
pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig {
index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS),
accounts_hash_cache_path: None,
filler_account_count: None,
filler_accounts_config: FillerAccountsConfig::const_default(),
hash_calc_num_passes: None,
write_cache_limit_bytes: None,
};
@ -148,11 +149,31 @@ pub struct AccountsAddRootTiming {
pub store_us: u64,
}
#[derive(Debug, Clone, Copy)]
pub struct FillerAccountsConfig {
/// Number of filler accounts
pub count: usize,
/// Data size per account, in bytes
pub size: usize,
}
impl FillerAccountsConfig {
pub const fn const_default() -> Self {
Self { count: 0, size: 0 }
}
}
impl Default for FillerAccountsConfig {
fn default() -> Self {
Self::const_default()
}
}
#[derive(Debug, Default, Clone)]
pub struct AccountsDbConfig {
pub index: Option<AccountsIndexConfig>,
pub accounts_hash_cache_path: Option<PathBuf>,
pub filler_account_count: Option<usize>,
pub filler_accounts_config: FillerAccountsConfig,
pub hash_calc_num_passes: Option<usize>,
pub write_cache_limit_bytes: Option<u64>,
}
@ -1090,7 +1111,7 @@ pub struct AccountsDb {
/// GeyserPlugin accounts update notifier
accounts_update_notifier: Option<AccountsUpdateNotifier>,
filler_account_count: usize,
filler_accounts_config: FillerAccountsConfig,
pub filler_account_suffix: Option<Pubkey>,
active_stats: ActiveStats,
@ -1633,7 +1654,7 @@ impl AccountsDb {
dirty_stores: DashMap::default(),
zero_lamport_accounts_to_purge_after_full_snapshot: DashSet::default(),
accounts_update_notifier: None,
filler_account_count: 0,
filler_accounts_config: FillerAccountsConfig::default(),
filler_account_suffix: None,
num_hash_scan_passes,
}
@ -1677,11 +1698,13 @@ impl AccountsDb {
let accounts_hash_cache_path = accounts_db_config
.as_ref()
.and_then(|x| x.accounts_hash_cache_path.clone());
let filler_account_count = accounts_db_config
let filler_accounts_config = accounts_db_config
.as_ref()
.and_then(|cfg| cfg.filler_account_count)
.map(|config| config.filler_accounts_config)
.unwrap_or_default();
let filler_account_suffix = if filler_account_count > 0 {
let filler_account_suffix = if filler_accounts_config.count > 0 {
Some(solana_sdk::pubkey::new_rand())
} else {
None
@ -1694,7 +1717,7 @@ impl AccountsDb {
caching_enabled,
shrink_ratio,
accounts_update_notifier,
filler_account_count,
filler_accounts_config,
filler_account_suffix,
write_cache_limit_bytes: accounts_db_config
.as_ref()
@ -5741,7 +5764,7 @@ impl AccountsDb {
};
let hash = AccountsHash {
filler_account_suffix: if self.filler_account_count > 0 {
filler_account_suffix: if self.filler_accounts_config.count > 0 {
self.filler_account_suffix
} else {
None
@ -6857,15 +6880,18 @@ impl AccountsDb {
/// The filler accounts are added to each slot in the snapshot after index generation.
/// The accounts added in a slot are setup to have pubkeys such that rent will be collected from them before (or when?) their slot becomes an epoch old.
/// Thus, the filler accounts are rewritten by rent and the old slot can be thrown away successfully.
pub fn maybe_add_filler_accounts(&self, epoch_schedule: &EpochSchedule) {
if self.filler_account_count == 0 {
pub fn maybe_add_filler_accounts(&self, epoch_schedule: &EpochSchedule, rent: &Rent) {
if self.filler_accounts_config.count == 0 {
return;
}
let max_root_inclusive = self.accounts_index.max_root_inclusive();
let epoch = epoch_schedule.get_epoch(max_root_inclusive);
info!("adding {} filler accounts", self.filler_account_count);
info!(
"adding {} filler accounts with size {}",
self.filler_accounts_config.count, self.filler_accounts_config.size,
);
// break this up to force the accounts out of memory after each pass
let passes = 100;
let mut roots = self.storage.all_slots();
@ -6879,8 +6905,9 @@ impl AccountsDb {
let string = "FiLLERACCoUNTooooooooooooooooooooooooooooooo";
let hash = Hash::from_str(string).unwrap();
let owner = Pubkey::from_str(string).unwrap();
let lamports = 100_000_000;
let space = 0;
let space = self.filler_accounts_config.size;
let rent_exempt_reserve = rent.minimum_balance(space);
let lamports = rent_exempt_reserve;
let account = AccountSharedData::new(lamports, space, &owner);
let added = AtomicUsize::default();
for pass in 0..=passes {
@ -6907,8 +6934,8 @@ impl AccountsDb {
let subrange = crate::bank::Bank::pubkey_range_from_partition(partition);
let idx = overall_index.fetch_add(1, Ordering::Relaxed);
let filler_entries = (idx + 1) * self.filler_account_count / root_count
- idx * self.filler_account_count / root_count;
let filler_entries = (idx + 1) * self.filler_accounts_config.count / root_count
- idx * self.filler_accounts_config.count / root_count;
let accounts = (0..filler_entries)
.map(|_| {
let my_id = added.fetch_add(1, Ordering::Relaxed);

2
runtime/src/serde_snapshot.rs
View File

@ -642,7 +642,7 @@ where
genesis_config,
);
accounts_db.maybe_add_filler_accounts(&genesis_config.epoch_schedule);
accounts_db.maybe_add_filler_accounts(&genesis_config.epoch_schedule, &genesis_config.rent);
handle.join().unwrap();
measure_notify.stop();

21
validator/src/main.rs
View File

@ -42,8 +42,8 @@ use {
},
solana_runtime::{
accounts_db::{
AccountShrinkThreshold, AccountsDbConfig, DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE,
DEFAULT_ACCOUNTS_SHRINK_RATIO,
AccountShrinkThreshold, AccountsDbConfig, FillerAccountsConfig,
DEFAULT_ACCOUNTS_SHRINK_OPTIMIZE_TOTAL_SPACE, DEFAULT_ACCOUNTS_SHRINK_RATIO,
},
accounts_index::{
AccountIndex, AccountSecondaryIndexes, AccountSecondaryIndexesIncludeExclude,
@ -1600,7 +1600,16 @@ pub fn main() {
.value_name("COUNT")
.validator(is_parsable::<usize>)
.takes_value(true)
.default_value("0")
.help("How many accounts to add to stress the system. Accounts are ignored in operations related to correctness."))
.arg(Arg::with_name("accounts_filler_size")
.long("accounts-filler-size")
.value_name("BYTES")
.validator(is_parsable::<usize>)
.takes_value(true)
.default_value("0")
.requires("accounts_filler_count")
.help("Size per filler account in bytes."))
.arg(
Arg::with_name("accounts_db_test_hash_calculation")
.long("accounts-db-test-hash-calculation")
@ -2251,11 +2260,15 @@ pub fn main() {
.ok()
.map(|mb| mb * MB);
let filler_account_count = value_t!(matches, "accounts_filler_count", usize).ok();
let filler_accounts_config = FillerAccountsConfig {
count: value_t_or_exit!(matches, "accounts_filler_count", usize),
size: value_t_or_exit!(matches, "accounts_filler_size", usize),
};
let mut accounts_db_config = AccountsDbConfig {
index: Some(accounts_index_config),
accounts_hash_cache_path: Some(ledger_path.clone()),
filler_account_count,
filler_accounts_config,
write_cache_limit_bytes: value_t!(matches, "accounts_db_cache_limit_mb", u64)
.ok()
.map(|mb| mb * MB as u64),