blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Serialize lamports per signature (#25364) 

* Serialize lamports per signature

* Add full snapshot archive test, enable features in previous tests
This commit is contained in:
Ashwin Sekar 2022-06-06 13:07:02 -07:00 committed by GitHub
parent ec64d5261f
commit 8caced68ce
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 320 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
runtime/src/bank.rs
View File

@ -1282,10 +1282,10 @@ pub struct Bank {
/// Deprecated, do not use
/// Latest transaction fees for transactions processed by this bank
fee_calculator: FeeCalculator,
pub(crate) fee_calculator: FeeCalculator,
/// Track cluster signature throughput and adjust fee rate
fee_rate_governor: FeeRateGovernor,
pub(crate) fee_rate_governor: FeeRateGovernor,
/// Rent that has been collected
collected_rent: AtomicU64,

22
runtime/src/genesis_utils.rs
View File

@ -196,18 +196,22 @@ pub fn create_genesis_config_with_leader(
pub fn activate_all_features(genesis_config: &mut GenesisConfig) {
// Activate all features at genesis in development mode
for feature_id in FeatureSet::default().inactive {
genesis_config.accounts.insert(
feature_id,
Account::from(feature::create_account(
&Feature {
activated_at: Some(0),
},
std::cmp::max(genesis_config.rent.minimum_balance(Feature::size_of()), 1),
)),
);
activate_feature(genesis_config, feature_id);
}
}
pub fn activate_feature(genesis_config: &mut GenesisConfig, feature_id: Pubkey) {
genesis_config.accounts.insert(
feature_id,
Account::from(feature::create_account(
&Feature {
activated_at: Some(0),
},
std::cmp::max(genesis_config.rent.minimum_balance(Feature::size_of()), 1),
)),
);
}
#[allow(clippy::too_many_arguments)]
pub fn create_genesis_config_with_leader_ex(
mint_lamports: u64,

72
runtime/src/serde_snapshot.rs
View File

@ -151,6 +151,14 @@ trait TypeContext<'a>: PartialEq {
where
Self: std::marker::Sized;
#[cfg(test)]
fn serialize_bank_and_storage_without_extra_fields<S: serde::ser::Serializer>(
serializer: S,
serializable_bank: &SerializableBankAndStorageNoExtra<'a, Self>,
) -> std::result::Result<S::Ok, S::Error>
where
Self: std::marker::Sized;
fn serialize_accounts_db_fields<S: serde::ser::Serializer>(
serializer: S,
serializable_db: &SerializableAccountsDb<'a, Self>,
@ -315,6 +323,37 @@ where
})
}
#[cfg(test)]
pub(crate) fn bank_to_stream_no_extra_fields<W>(
serde_style: SerdeStyle,
stream: &mut BufWriter<W>,
bank: &Bank,
snapshot_storages: &[SnapshotStorage],
) -> Result<(), Error>
where
W: Write,
{
macro_rules! INTO {
($style:ident) => {
bincode::serialize_into(
stream,
&SerializableBankAndStorageNoExtra::<$style::Context> {
bank,
snapshot_storages,
phantom: std::marker::PhantomData::default(),
},
)
};
}
match serde_style {
SerdeStyle::Newer => INTO!(newer),
}
.map_err(|err| {
warn!("bankrc_to_stream error: {:?}", err);
err
})
}
/// deserialize the bank from 'stream_reader'
/// modify the accounts_hash
/// reserialize the bank to 'stream_writer'
@ -381,6 +420,39 @@ impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorage<'a, C> {
}
}
#[cfg(test)]
struct SerializableBankAndStorageNoExtra<'a, C> {
bank: &'a Bank,
snapshot_storages: &'a [SnapshotStorage],
phantom: std::marker::PhantomData<C>,
}
#[cfg(test)]
impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorageNoExtra<'a, C> {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::ser::Serializer,
{
C::serialize_bank_and_storage_without_extra_fields(serializer, self)
}
}
#[cfg(test)]
impl<'a, C> From<SerializableBankAndStorageNoExtra<'a, C>> for SerializableBankAndStorage<'a, C> {
fn from(s: SerializableBankAndStorageNoExtra<'a, C>) -> SerializableBankAndStorage<'a, C> {
let SerializableBankAndStorageNoExtra {
bank,
snapshot_storages,
phantom,
} = s;
SerializableBankAndStorage {
bank,
snapshot_storages,
phantom,
}
}
}
struct SerializableAccountsDb<'a, C> {
accounts_db: &'a AccountsDb,
slot: Slot,

45
runtime/src/serde_snapshot/newer.rs
View File

@ -191,6 +191,33 @@ impl<'a> TypeContext<'a> for Context {
serializer: S,
serializable_bank: &SerializableBankAndStorage<'a, Self>,
) -> std::result::Result<S::Ok, S::Error>
where
Self: std::marker::Sized,
{
let ancestors = HashMap::from(&serializable_bank.bank.ancestors);
let fields = serializable_bank.bank.get_fields_to_serialize(&ancestors);
let lamports_per_signature = fields.fee_rate_governor.lamports_per_signature;
(
SerializableVersionedBank::from(fields),
SerializableAccountsDb::<'a, Self> {
accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db,
slot: serializable_bank.bank.rc.slot,
account_storage_entries: serializable_bank.snapshot_storages,
phantom: std::marker::PhantomData::default(),
},
// Additional fields, we manually store the lamps per signature here so that
// we can grab it on restart.
// TODO: if we do a snapshot version bump, consider moving this out.
lamports_per_signature,
)
.serialize(serializer)
}
#[cfg(test)]
fn serialize_bank_and_storage_without_extra_fields<S: serde::ser::Serializer>(
serializer: S,
serializable_bank: &SerializableBankAndStorageNoExtra<'a, Self>,
) -> std::result::Result<S::Ok, S::Error>
where
Self: std::marker::Sized,
{
@ -279,8 +306,18 @@ impl<'a> TypeContext<'a> for Context {
where
R: Read,
{
let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into();
let mut bank_fields: BankFieldsToDeserialize =
deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into();
let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?;
// Process extra fields
let lamports_per_signature: u64 = match deserialize_from(stream) {
Err(err) if err.to_string() == "io error: unexpected end of file" => Ok(0),
Err(err) if err.to_string() == "io error: failed to fill whole buffer" => Ok(0),
result => result,
}?;
bank_fields.fee_rate_governor = bank_fields
.fee_rate_governor
.clone_with_lamports_per_signature(lamports_per_signature);
Ok((bank_fields, accounts_db_fields))
}
@ -311,6 +348,7 @@ impl<'a> TypeContext<'a> for Context {
let rhs = bank_fields;
let blockhash_queue = RwLock::new(rhs.blockhash_queue.clone());
let hard_forks = RwLock::new(rhs.hard_forks.clone());
let lamports_per_signature = rhs.fee_rate_governor.lamports_per_signature;
let bank = SerializableVersionedBank {
blockhash_queue: &blockhash_queue,
ancestors: &rhs.ancestors,
@ -346,6 +384,9 @@ impl<'a> TypeContext<'a> for Context {
is_delta: rhs.is_delta,
};
bincode::serialize_into(stream_writer, &(bank, accounts_db_fields))
bincode::serialize_into(
stream_writer,
&(bank, accounts_db_fields, lamports_per_signature),
)
}
}

184
runtime/src/serde_snapshot/tests.rs
View File

@ -5,13 +5,16 @@ use {
accounts::{test_utils::create_test_accounts, Accounts},
accounts_db::{get_temp_accounts_paths, AccountShrinkThreshold},
bank::{Bank, Rewrites, StatusCacheRc},
genesis_utils::{activate_all_features, activate_feature},
hardened_unpack::UnpackedAppendVecMap,
snapshot_utils::ArchiveFormat,
},
bincode::serialize_into,
rand::{thread_rng, Rng},
solana_sdk::{
account::{AccountSharedData, ReadableAccount},
clock::Slot,
feature_set::disable_fee_calculator,
genesis_config::{create_genesis_config, ClusterType},
pubkey::Pubkey,
signature::{Keypair, Signer},
@ -361,13 +364,192 @@ fn test_bank_serialize_newer() {
}
}
#[test]
fn test_extra_fields_eof() {
solana_logger::setup();
let (mut genesis_config, _) = create_genesis_config(500);
activate_feature(&mut genesis_config, disable_fee_calculator::id());
let bank0 = Arc::new(Bank::new_for_tests(&genesis_config));
bank0.squash();
let mut bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
// Set extra fields
bank.fee_rate_governor.lamports_per_signature = 7000;
// Serialize
let snapshot_storages = bank.get_snapshot_storages(None);
let mut buf = vec![];
let mut writer = Cursor::new(&mut buf);
crate::serde_snapshot::bank_to_stream(
SerdeStyle::Newer,
&mut std::io::BufWriter::new(&mut writer),
&bank,
&snapshot_storages,
)
.unwrap();
// Deserialize
let rdr = Cursor::new(&buf[..]);
let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]);
let mut snapshot_streams = SnapshotStreams {
full_snapshot_stream: &mut reader,
incremental_snapshot_stream: None,
};
let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap();
let copied_accounts = TempDir::new().unwrap();
let unpacked_append_vec_map =
copy_append_vecs(&bank.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
let dbank = crate::serde_snapshot::bank_from_streams(
SerdeStyle::Newer,
&mut snapshot_streams,
&dbank_paths,
unpacked_append_vec_map,
&genesis_config,
None,
None,
AccountSecondaryIndexes::default(),
false,
None,
AccountShrinkThreshold::default(),
false,
Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
.unwrap();
assert_eq!(
bank.fee_rate_governor.lamports_per_signature,
dbank.fee_rate_governor.lamports_per_signature
);
}
#[test]
fn test_extra_fields_full_snapshot_archive() {
solana_logger::setup();
let (mut genesis_config, _) = create_genesis_config(500);
activate_all_features(&mut genesis_config);
let bank0 = Arc::new(Bank::new_for_tests(&genesis_config));
let mut bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
while !bank.is_complete() {
bank.fill_bank_with_ticks_for_tests();
}
// Set extra field
bank.fee_rate_governor.lamports_per_signature = 7000;
let accounts_dir = TempDir::new().unwrap();
let bank_snapshots_dir = TempDir::new().unwrap();
let full_snapshot_archives_dir = TempDir::new().unwrap();
let incremental_snapshot_archives_dir = TempDir::new().unwrap();
// Serialize
let snapshot_archive_info = snapshot_utils::bank_to_full_snapshot_archive(
&bank_snapshots_dir,
&bank,
None,
full_snapshot_archives_dir.path(),
incremental_snapshot_archives_dir.path(),
ArchiveFormat::TarBzip2,
1,
0,
)
.unwrap();
// Deserialize
let (dbank, _) = snapshot_utils::bank_from_snapshot_archives(
&[PathBuf::from(accounts_dir.path())],
bank_snapshots_dir.path(),
&snapshot_archive_info,
None,
&genesis_config,
None,
None,
AccountSecondaryIndexes::default(),
false,
None,
AccountShrinkThreshold::default(),
false,
false,
false,
Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
.unwrap();
assert_eq!(
bank.fee_rate_governor.lamports_per_signature,
dbank.fee_rate_governor.lamports_per_signature
);
}
#[test]
fn test_blank_extra_fields() {
solana_logger::setup();
let (mut genesis_config, _) = create_genesis_config(500);
activate_feature(&mut genesis_config, disable_fee_calculator::id());
let bank0 = Arc::new(Bank::new_for_tests(&genesis_config));
bank0.squash();
let mut bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
// Set extra fields
bank.fee_rate_governor.lamports_per_signature = 7000;
// Serialize, but don't serialize the extra fields
let snapshot_storages = bank.get_snapshot_storages(None);
let mut buf = vec![];
let mut writer = Cursor::new(&mut buf);
crate::serde_snapshot::bank_to_stream_no_extra_fields(
SerdeStyle::Newer,
&mut std::io::BufWriter::new(&mut writer),
&bank,
&snapshot_storages,
)
.unwrap();
// Deserialize
let rdr = Cursor::new(&buf[..]);
let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]);
let mut snapshot_streams = SnapshotStreams {
full_snapshot_stream: &mut reader,
incremental_snapshot_stream: None,
};
let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap();
let copied_accounts = TempDir::new().unwrap();
let unpacked_append_vec_map =
copy_append_vecs(&bank.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
let dbank = crate::serde_snapshot::bank_from_streams(
SerdeStyle::Newer,
&mut snapshot_streams,
&dbank_paths,
unpacked_append_vec_map,
&genesis_config,
None,
None,
AccountSecondaryIndexes::default(),
false,
None,
AccountShrinkThreshold::default(),
false,
Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING),
None,
)
.unwrap();
// Defaults to 0
assert_eq!(0, dbank.fee_rate_governor.lamports_per_signature);
}
#[cfg(RUSTC_WITH_SPECIALIZATION)]
mod test_bank_serialize {
use super::*;
// This some what long test harness is required to freeze the ABI of
// Bank's serialization due to versioned nature
#[frozen_abi(digest = "HT9yewU4zJ6ZAgJ7aDSbHPtzZGZqASpq6rkq6ET42Kki")]
#[frozen_abi(digest = "9vGBt7YfymKUTPWLHVVpQbDtPD7dFDwXRMFkCzwujNqJ")]
#[derive(Serialize, AbiExample)]
pub struct BankAbiTestWrapperNewer {
#[serde(serialize_with = "wrapper_newer")]

7
sdk/program/src/fee_calculator.rs
View File

@ -165,6 +165,13 @@ impl FeeRateGovernor {
me
}
pub fn clone_with_lamports_per_signature(&self, lamports_per_signature: u64) -> Self {
Self {
lamports_per_signature,
..*self
}
}
/// calculate unburned fee from a fee total, returns (unburned, burned)
pub fn burn(&self, fees: u64) -> (u64, u64) {
let burned = fees * u64::from(self.burn_percent) / 100;