Serialize lamports per signature (#25364)
* Serialize lamports per signature * Add full snapshot archive test, enable features in previous tests
This commit is contained in:
parent
ec64d5261f
commit
8caced68ce
|
@ -1282,10 +1282,10 @@ pub struct Bank {
|
|||
|
||||
/// Deprecated, do not use
|
||||
/// Latest transaction fees for transactions processed by this bank
|
||||
fee_calculator: FeeCalculator,
|
||||
pub(crate) fee_calculator: FeeCalculator,
|
||||
|
||||
/// Track cluster signature throughput and adjust fee rate
|
||||
fee_rate_governor: FeeRateGovernor,
|
||||
pub(crate) fee_rate_governor: FeeRateGovernor,
|
||||
|
||||
/// Rent that has been collected
|
||||
collected_rent: AtomicU64,
|
||||
|
|
|
@ -196,6 +196,11 @@ pub fn create_genesis_config_with_leader(
|
|||
pub fn activate_all_features(genesis_config: &mut GenesisConfig) {
|
||||
// Activate all features at genesis in development mode
|
||||
for feature_id in FeatureSet::default().inactive {
|
||||
activate_feature(genesis_config, feature_id);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn activate_feature(genesis_config: &mut GenesisConfig, feature_id: Pubkey) {
|
||||
genesis_config.accounts.insert(
|
||||
feature_id,
|
||||
Account::from(feature::create_account(
|
||||
|
@ -205,7 +210,6 @@ pub fn activate_all_features(genesis_config: &mut GenesisConfig) {
|
|||
std::cmp::max(genesis_config.rent.minimum_balance(Feature::size_of()), 1),
|
||||
)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
|
|
|
@ -151,6 +151,14 @@ trait TypeContext<'a>: PartialEq {
|
|||
where
|
||||
Self: std::marker::Sized;
|
||||
|
||||
#[cfg(test)]
|
||||
fn serialize_bank_and_storage_without_extra_fields<S: serde::ser::Serializer>(
|
||||
serializer: S,
|
||||
serializable_bank: &SerializableBankAndStorageNoExtra<'a, Self>,
|
||||
) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
Self: std::marker::Sized;
|
||||
|
||||
fn serialize_accounts_db_fields<S: serde::ser::Serializer>(
|
||||
serializer: S,
|
||||
serializable_db: &SerializableAccountsDb<'a, Self>,
|
||||
|
@ -315,6 +323,37 @@ where
|
|||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn bank_to_stream_no_extra_fields<W>(
|
||||
serde_style: SerdeStyle,
|
||||
stream: &mut BufWriter<W>,
|
||||
bank: &Bank,
|
||||
snapshot_storages: &[SnapshotStorage],
|
||||
) -> Result<(), Error>
|
||||
where
|
||||
W: Write,
|
||||
{
|
||||
macro_rules! INTO {
|
||||
($style:ident) => {
|
||||
bincode::serialize_into(
|
||||
stream,
|
||||
&SerializableBankAndStorageNoExtra::<$style::Context> {
|
||||
bank,
|
||||
snapshot_storages,
|
||||
phantom: std::marker::PhantomData::default(),
|
||||
},
|
||||
)
|
||||
};
|
||||
}
|
||||
match serde_style {
|
||||
SerdeStyle::Newer => INTO!(newer),
|
||||
}
|
||||
.map_err(|err| {
|
||||
warn!("bankrc_to_stream error: {:?}", err);
|
||||
err
|
||||
})
|
||||
}
|
||||
|
||||
/// deserialize the bank from 'stream_reader'
|
||||
/// modify the accounts_hash
|
||||
/// reserialize the bank to 'stream_writer'
|
||||
|
@ -381,6 +420,39 @@ impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorage<'a, C> {
|
|||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct SerializableBankAndStorageNoExtra<'a, C> {
|
||||
bank: &'a Bank,
|
||||
snapshot_storages: &'a [SnapshotStorage],
|
||||
phantom: std::marker::PhantomData<C>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl<'a, C: TypeContext<'a>> Serialize for SerializableBankAndStorageNoExtra<'a, C> {
|
||||
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::ser::Serializer,
|
||||
{
|
||||
C::serialize_bank_and_storage_without_extra_fields(serializer, self)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl<'a, C> From<SerializableBankAndStorageNoExtra<'a, C>> for SerializableBankAndStorage<'a, C> {
|
||||
fn from(s: SerializableBankAndStorageNoExtra<'a, C>) -> SerializableBankAndStorage<'a, C> {
|
||||
let SerializableBankAndStorageNoExtra {
|
||||
bank,
|
||||
snapshot_storages,
|
||||
phantom,
|
||||
} = s;
|
||||
SerializableBankAndStorage {
|
||||
bank,
|
||||
snapshot_storages,
|
||||
phantom,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct SerializableAccountsDb<'a, C> {
|
||||
accounts_db: &'a AccountsDb,
|
||||
slot: Slot,
|
||||
|
|
|
@ -191,6 +191,33 @@ impl<'a> TypeContext<'a> for Context {
|
|||
serializer: S,
|
||||
serializable_bank: &SerializableBankAndStorage<'a, Self>,
|
||||
) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
Self: std::marker::Sized,
|
||||
{
|
||||
let ancestors = HashMap::from(&serializable_bank.bank.ancestors);
|
||||
let fields = serializable_bank.bank.get_fields_to_serialize(&ancestors);
|
||||
let lamports_per_signature = fields.fee_rate_governor.lamports_per_signature;
|
||||
(
|
||||
SerializableVersionedBank::from(fields),
|
||||
SerializableAccountsDb::<'a, Self> {
|
||||
accounts_db: &*serializable_bank.bank.rc.accounts.accounts_db,
|
||||
slot: serializable_bank.bank.rc.slot,
|
||||
account_storage_entries: serializable_bank.snapshot_storages,
|
||||
phantom: std::marker::PhantomData::default(),
|
||||
},
|
||||
// Additional fields, we manually store the lamps per signature here so that
|
||||
// we can grab it on restart.
|
||||
// TODO: if we do a snapshot version bump, consider moving this out.
|
||||
lamports_per_signature,
|
||||
)
|
||||
.serialize(serializer)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn serialize_bank_and_storage_without_extra_fields<S: serde::ser::Serializer>(
|
||||
serializer: S,
|
||||
serializable_bank: &SerializableBankAndStorageNoExtra<'a, Self>,
|
||||
) -> std::result::Result<S::Ok, S::Error>
|
||||
where
|
||||
Self: std::marker::Sized,
|
||||
{
|
||||
|
@ -279,8 +306,18 @@ impl<'a> TypeContext<'a> for Context {
|
|||
where
|
||||
R: Read,
|
||||
{
|
||||
let bank_fields = deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into();
|
||||
let mut bank_fields: BankFieldsToDeserialize =
|
||||
deserialize_from::<_, DeserializableVersionedBank>(&mut stream)?.into();
|
||||
let accounts_db_fields = Self::deserialize_accounts_db_fields(stream)?;
|
||||
// Process extra fields
|
||||
let lamports_per_signature: u64 = match deserialize_from(stream) {
|
||||
Err(err) if err.to_string() == "io error: unexpected end of file" => Ok(0),
|
||||
Err(err) if err.to_string() == "io error: failed to fill whole buffer" => Ok(0),
|
||||
result => result,
|
||||
}?;
|
||||
bank_fields.fee_rate_governor = bank_fields
|
||||
.fee_rate_governor
|
||||
.clone_with_lamports_per_signature(lamports_per_signature);
|
||||
Ok((bank_fields, accounts_db_fields))
|
||||
}
|
||||
|
||||
|
@ -311,6 +348,7 @@ impl<'a> TypeContext<'a> for Context {
|
|||
let rhs = bank_fields;
|
||||
let blockhash_queue = RwLock::new(rhs.blockhash_queue.clone());
|
||||
let hard_forks = RwLock::new(rhs.hard_forks.clone());
|
||||
let lamports_per_signature = rhs.fee_rate_governor.lamports_per_signature;
|
||||
let bank = SerializableVersionedBank {
|
||||
blockhash_queue: &blockhash_queue,
|
||||
ancestors: &rhs.ancestors,
|
||||
|
@ -346,6 +384,9 @@ impl<'a> TypeContext<'a> for Context {
|
|||
is_delta: rhs.is_delta,
|
||||
};
|
||||
|
||||
bincode::serialize_into(stream_writer, &(bank, accounts_db_fields))
|
||||
bincode::serialize_into(
|
||||
stream_writer,
|
||||
&(bank, accounts_db_fields, lamports_per_signature),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -5,13 +5,16 @@ use {
|
|||
accounts::{test_utils::create_test_accounts, Accounts},
|
||||
accounts_db::{get_temp_accounts_paths, AccountShrinkThreshold},
|
||||
bank::{Bank, Rewrites, StatusCacheRc},
|
||||
genesis_utils::{activate_all_features, activate_feature},
|
||||
hardened_unpack::UnpackedAppendVecMap,
|
||||
snapshot_utils::ArchiveFormat,
|
||||
},
|
||||
bincode::serialize_into,
|
||||
rand::{thread_rng, Rng},
|
||||
solana_sdk::{
|
||||
account::{AccountSharedData, ReadableAccount},
|
||||
clock::Slot,
|
||||
feature_set::disable_fee_calculator,
|
||||
genesis_config::{create_genesis_config, ClusterType},
|
||||
pubkey::Pubkey,
|
||||
signature::{Keypair, Signer},
|
||||
|
@ -361,13 +364,192 @@ fn test_bank_serialize_newer() {
|
|||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_fields_eof() {
|
||||
solana_logger::setup();
|
||||
let (mut genesis_config, _) = create_genesis_config(500);
|
||||
activate_feature(&mut genesis_config, disable_fee_calculator::id());
|
||||
|
||||
let bank0 = Arc::new(Bank::new_for_tests(&genesis_config));
|
||||
bank0.squash();
|
||||
let mut bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
|
||||
|
||||
// Set extra fields
|
||||
bank.fee_rate_governor.lamports_per_signature = 7000;
|
||||
|
||||
// Serialize
|
||||
let snapshot_storages = bank.get_snapshot_storages(None);
|
||||
let mut buf = vec![];
|
||||
let mut writer = Cursor::new(&mut buf);
|
||||
crate::serde_snapshot::bank_to_stream(
|
||||
SerdeStyle::Newer,
|
||||
&mut std::io::BufWriter::new(&mut writer),
|
||||
&bank,
|
||||
&snapshot_storages,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Deserialize
|
||||
let rdr = Cursor::new(&buf[..]);
|
||||
let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]);
|
||||
let mut snapshot_streams = SnapshotStreams {
|
||||
full_snapshot_stream: &mut reader,
|
||||
incremental_snapshot_stream: None,
|
||||
};
|
||||
let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap();
|
||||
let copied_accounts = TempDir::new().unwrap();
|
||||
let unpacked_append_vec_map =
|
||||
copy_append_vecs(&bank.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
|
||||
let dbank = crate::serde_snapshot::bank_from_streams(
|
||||
SerdeStyle::Newer,
|
||||
&mut snapshot_streams,
|
||||
&dbank_paths,
|
||||
unpacked_append_vec_map,
|
||||
&genesis_config,
|
||||
None,
|
||||
None,
|
||||
AccountSecondaryIndexes::default(),
|
||||
false,
|
||||
None,
|
||||
AccountShrinkThreshold::default(),
|
||||
false,
|
||||
Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
bank.fee_rate_governor.lamports_per_signature,
|
||||
dbank.fee_rate_governor.lamports_per_signature
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_extra_fields_full_snapshot_archive() {
|
||||
solana_logger::setup();
|
||||
|
||||
let (mut genesis_config, _) = create_genesis_config(500);
|
||||
activate_all_features(&mut genesis_config);
|
||||
|
||||
let bank0 = Arc::new(Bank::new_for_tests(&genesis_config));
|
||||
let mut bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
|
||||
while !bank.is_complete() {
|
||||
bank.fill_bank_with_ticks_for_tests();
|
||||
}
|
||||
|
||||
// Set extra field
|
||||
bank.fee_rate_governor.lamports_per_signature = 7000;
|
||||
|
||||
let accounts_dir = TempDir::new().unwrap();
|
||||
let bank_snapshots_dir = TempDir::new().unwrap();
|
||||
let full_snapshot_archives_dir = TempDir::new().unwrap();
|
||||
let incremental_snapshot_archives_dir = TempDir::new().unwrap();
|
||||
|
||||
// Serialize
|
||||
let snapshot_archive_info = snapshot_utils::bank_to_full_snapshot_archive(
|
||||
&bank_snapshots_dir,
|
||||
&bank,
|
||||
None,
|
||||
full_snapshot_archives_dir.path(),
|
||||
incremental_snapshot_archives_dir.path(),
|
||||
ArchiveFormat::TarBzip2,
|
||||
1,
|
||||
0,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Deserialize
|
||||
let (dbank, _) = snapshot_utils::bank_from_snapshot_archives(
|
||||
&[PathBuf::from(accounts_dir.path())],
|
||||
bank_snapshots_dir.path(),
|
||||
&snapshot_archive_info,
|
||||
None,
|
||||
&genesis_config,
|
||||
None,
|
||||
None,
|
||||
AccountSecondaryIndexes::default(),
|
||||
false,
|
||||
None,
|
||||
AccountShrinkThreshold::default(),
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
bank.fee_rate_governor.lamports_per_signature,
|
||||
dbank.fee_rate_governor.lamports_per_signature
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_blank_extra_fields() {
|
||||
solana_logger::setup();
|
||||
let (mut genesis_config, _) = create_genesis_config(500);
|
||||
activate_feature(&mut genesis_config, disable_fee_calculator::id());
|
||||
|
||||
let bank0 = Arc::new(Bank::new_for_tests(&genesis_config));
|
||||
bank0.squash();
|
||||
let mut bank = Bank::new_from_parent(&bank0, &Pubkey::default(), 1);
|
||||
|
||||
// Set extra fields
|
||||
bank.fee_rate_governor.lamports_per_signature = 7000;
|
||||
|
||||
// Serialize, but don't serialize the extra fields
|
||||
let snapshot_storages = bank.get_snapshot_storages(None);
|
||||
let mut buf = vec![];
|
||||
let mut writer = Cursor::new(&mut buf);
|
||||
crate::serde_snapshot::bank_to_stream_no_extra_fields(
|
||||
SerdeStyle::Newer,
|
||||
&mut std::io::BufWriter::new(&mut writer),
|
||||
&bank,
|
||||
&snapshot_storages,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Deserialize
|
||||
let rdr = Cursor::new(&buf[..]);
|
||||
let mut reader = std::io::BufReader::new(&buf[rdr.position() as usize..]);
|
||||
let mut snapshot_streams = SnapshotStreams {
|
||||
full_snapshot_stream: &mut reader,
|
||||
incremental_snapshot_stream: None,
|
||||
};
|
||||
let (_accounts_dir, dbank_paths) = get_temp_accounts_paths(4).unwrap();
|
||||
let copied_accounts = TempDir::new().unwrap();
|
||||
let unpacked_append_vec_map =
|
||||
copy_append_vecs(&bank.rc.accounts.accounts_db, copied_accounts.path()).unwrap();
|
||||
let dbank = crate::serde_snapshot::bank_from_streams(
|
||||
SerdeStyle::Newer,
|
||||
&mut snapshot_streams,
|
||||
&dbank_paths,
|
||||
unpacked_append_vec_map,
|
||||
&genesis_config,
|
||||
None,
|
||||
None,
|
||||
AccountSecondaryIndexes::default(),
|
||||
false,
|
||||
None,
|
||||
AccountShrinkThreshold::default(),
|
||||
false,
|
||||
Some(crate::accounts_db::ACCOUNTS_DB_CONFIG_FOR_TESTING),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Defaults to 0
|
||||
assert_eq!(0, dbank.fee_rate_governor.lamports_per_signature);
|
||||
}
|
||||
|
||||
#[cfg(RUSTC_WITH_SPECIALIZATION)]
|
||||
mod test_bank_serialize {
|
||||
use super::*;
|
||||
|
||||
// This some what long test harness is required to freeze the ABI of
|
||||
// Bank's serialization due to versioned nature
|
||||
#[frozen_abi(digest = "HT9yewU4zJ6ZAgJ7aDSbHPtzZGZqASpq6rkq6ET42Kki")]
|
||||
#[frozen_abi(digest = "9vGBt7YfymKUTPWLHVVpQbDtPD7dFDwXRMFkCzwujNqJ")]
|
||||
#[derive(Serialize, AbiExample)]
|
||||
pub struct BankAbiTestWrapperNewer {
|
||||
#[serde(serialize_with = "wrapper_newer")]
|
||||
|
|
|
@ -165,6 +165,13 @@ impl FeeRateGovernor {
|
|||
me
|
||||
}
|
||||
|
||||
pub fn clone_with_lamports_per_signature(&self, lamports_per_signature: u64) -> Self {
|
||||
Self {
|
||||
lamports_per_signature,
|
||||
..*self
|
||||
}
|
||||
}
|
||||
|
||||
/// calculate unburned fee from a fee total, returns (unburned, burned)
|
||||
pub fn burn(&self, fees: u64) -> (u64, u64) {
|
||||
let burned = fees * u64::from(self.burn_percent) / 100;
|
||||
|
|
Loading…
Reference in New Issue