first complete tests on token program accounts

This commit is contained in:
godmodegalactus 2024-07-04 18:24:16 +02:00
parent 5114a9b379
commit be165f600d
No known key found for this signature in database
GPG Key ID: 22DA4A30887FDA3C
8 changed files with 658 additions and 24 deletions

68
Cargo.lock generated
View File

@ -1943,6 +1943,7 @@ dependencies = [
"spl-token",
"spl-token-2022 3.0.2",
"tokio",
"tracing-subscriber",
]
[[package]]
@ -2081,6 +2082,16 @@ dependencies = [
"minimal-lexical",
]
[[package]]
name = "nu-ansi-term"
version = "0.46.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84"
dependencies = [
"overload",
"winapi",
]
[[package]]
name = "num"
version = "0.2.1"
@ -2295,6 +2306,12 @@ version = "6.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"
[[package]]
name = "overload"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39"
[[package]]
name = "parking_lot"
version = "0.12.3"
@ -3136,6 +3153,15 @@ dependencies = [
"keccak",
]
[[package]]
name = "sharded-slab"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
dependencies = [
"lazy_static",
]
[[package]]
name = "shell-words"
version = "1.1.0"
@ -4479,6 +4505,16 @@ dependencies = [
"syn 2.0.68",
]
[[package]]
name = "thread_local"
version = "1.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c"
dependencies = [
"cfg-if",
"once_cell",
]
[[package]]
name = "time"
version = "0.3.36"
@ -4696,6 +4732,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
dependencies = [
"once_cell",
"valuable",
]
[[package]]
name = "tracing-log"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
dependencies = [
"log",
"once_cell",
"tracing-core",
]
[[package]]
name = "tracing-subscriber"
version = "0.3.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b"
dependencies = [
"nu-ansi-term",
"sharded-slab",
"smallvec",
"thread_local",
"tracing-core",
"tracing-log",
]
[[package]]
@ -4822,6 +4884,12 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]]
name = "valuable"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "vec_map"
version = "0.8.2"

View File

@ -40,6 +40,7 @@ lazy_static = "1.4.0"
async-trait = "0.1.68"
base64 = "0.21.0"
futures = "0.3.28"
tracing-subscriber = "0.3.16"
# spl token
spl-token = "~4.0.0"

View File

@ -1,5 +1,6 @@
use solana_sdk::clock::Slot;
#[derive(PartialEq, Eq, Debug, Clone, Copy)]
pub struct SlotInfo {
pub slot: Slot,
pub parent: Slot,

View File

@ -23,6 +23,8 @@ anyhow = {workspace = true}
bincode = {workspace = true}
lz4 = {workspace = true}
[dev-dependencies]
rand = "0.8.5"
rand_chacha = "0.3.1"
rand_chacha = "0.3.1"
tracing-subscriber = {workspace = true}

View File

@ -63,6 +63,11 @@ impl ProcessedAccountStore {
{
Some(acc) => {
if acc.write_version < write_version {
log::debug!(
"updating an existing account {account_pk:?} from {} to {}",
acc.write_version,
write_version
);
acc.write_version = write_version;
acc.processed_account = processed_account;
true
@ -71,6 +76,7 @@ impl ProcessedAccountStore {
}
}
None => {
log::debug!("Adding a new account {account_pk:?} to slot {slot}");
processed_account_by_slot.processed_accounts.insert(
account_pk,
ProcessedAccount {
@ -83,6 +89,7 @@ impl ProcessedAccountStore {
}
}
None => {
log::debug!("Adding a new slot {slot} with new account {account_pk:?}");
let mut processed_accounts = HashMap::new();
processed_accounts.insert(
account_pk,
@ -104,6 +111,12 @@ impl ProcessedAccountStore {
}
pub async fn update_slot_info(&self, slot_info: SlotInfo) {
log::debug!(
"update slot info slot : {}, parent: {}, root: {}",
slot_info.slot,
slot_info.parent,
slot_info.root
);
let mut lk = self.processed_slot_accounts.write().await;
match lk.get_mut(&slot_info.slot) {
Some(processed_account_by_slot) => {
@ -150,8 +163,9 @@ impl ProcessedAccountStore {
}
// returns list of accounts that are finalized
pub async fn finalize(&self, slot: u64) -> Vec<ProcessedAccount> {
let mut map_of_accounts: HashMap<Pubkey, ProcessedAccount> = HashMap::new();
pub async fn finalize(&self, slot: u64) -> Vec<(ProcessedAccount, u64)> {
log::debug!("finalizing slot : {slot}");
let mut map_of_accounts: HashMap<Pubkey, (ProcessedAccount, u64)> = HashMap::new();
let mut lk = self.processed_slot_accounts.write().await;
let mut process_slot = Some(slot);
while let Some(current_slot) = process_slot {
@ -163,7 +177,7 @@ impl ProcessedAccountStore {
// if the key already exists then we do not insert in the map
match map_of_accounts.entry(pk) {
std::collections::hash_map::Entry::Vacant(vac) => {
vac.insert(acc);
vac.insert((acc, current_slot));
}
std::collections::hash_map::Entry::Occupied(_) => {
// already present
@ -198,7 +212,7 @@ impl ProcessedAccountStore {
pub async fn get_confirmed_accounts(
&self,
account_pks: Vec<Pubkey>,
return_list: &mut [Option<ProcessedAccount>],
return_list: &mut [Option<(ProcessedAccount, u64)>],
confirmed_slot: Slot,
) {
let mut process_slot = Some(confirmed_slot);
@ -211,7 +225,8 @@ impl ProcessedAccountStore {
continue;
}
if let Some(acc) = processed_accounts.processed_accounts.get(account_pk) {
*return_list.get_mut(index).unwrap() = Some(acc.clone());
*return_list.get_mut(index).unwrap() =
Some((acc.clone(), current_slot));
}
}
if return_list.iter().all(|x| x.is_some()) {
@ -231,13 +246,14 @@ impl ProcessedAccountStore {
account_pks: Vec<Pubkey>,
commitment: Commitment,
confirmed_slot: Slot,
) -> Vec<Option<ProcessedAccount>> {
) -> Vec<Option<(ProcessedAccount, u64)>> {
let mut return_list = vec![None; account_pks.len()];
match commitment {
Commitment::Processed => {
let lk = self.processed_slot_accounts.read().await;
// iterate backwards on all the processed slots until we reach confirmed slot
for (slot, processed_account_slots) in lk.iter().rev() {
log::debug!("getting processed account at slot : {}", slot);
if *slot > confirmed_slot {
for (index, account_pk) in account_pks.iter().enumerate() {
if return_list.get(index).unwrap().is_some() {
@ -247,7 +263,7 @@ impl ProcessedAccountStore {
processed_account_slots.processed_accounts.get(account_pk)
{
*return_list.get_mut(index).unwrap() =
Some(processed_account.clone());
Some((processed_account.clone(), *slot));
}
}
@ -519,6 +535,11 @@ impl AccountStorageInterface for InMemoryTokenStorage {
return false;
}
log::debug!(
"update account : {} for commitment : {}",
account_data.pubkey.to_string(),
commitment
);
let token_program_account = match get_token_program_account_type(
&account_data,
&self.mints_index_by_pubkey,
@ -549,6 +570,7 @@ impl AccountStorageInterface for InMemoryTokenStorage {
return;
}
log::debug!("initializing account : {}", account_data.pubkey.to_string());
// add account for finalized commitment
let token_program_account = match get_token_program_account_type(
&account_data,
@ -569,6 +591,7 @@ impl AccountStorageInterface for InMemoryTokenStorage {
account_pk: Pubkey,
commitment: Commitment,
) -> Result<Option<AccountData>, AccountLoadingError> {
log::debug!("get account : {}", account_pk.to_string());
match commitment {
Commitment::Confirmed | Commitment::Processed => {
match self
@ -582,9 +605,9 @@ impl AccountStorageInterface for InMemoryTokenStorage {
.get(0)
.unwrap()
{
Some(processed_account) => Ok(token_program_account_to_solana_account(
Some((processed_account, slot)) => Ok(token_program_account_to_solana_account(
&processed_account.processed_account,
self.confirmed_slot.load(Ordering::Relaxed),
*slot,
0,
&self.mints_by_index,
)),
@ -618,10 +641,10 @@ impl AccountStorageInterface for InMemoryTokenStorage {
.await;
let mut to_remove = vec![];
for (index, processed_account) in processed_accounts.iter().enumerate() {
if let Some(processed_account) = processed_account {
if let Some((processed_account, slot)) = processed_account {
let updated_account = token_program_account_to_solana_account(
&processed_account.processed_account,
confirmed_slot,
*slot,
processed_account.write_version,
&self.mints_by_index,
);
@ -652,31 +675,42 @@ impl AccountStorageInterface for InMemoryTokenStorage {
) -> Vec<AccountData> {
match commitment {
Commitment::Processed => {
log::debug!("process slot data : {slot_info:?}");
self.processed_storage.update_slot_info(slot_info).await;
vec![]
}
Commitment::Confirmed => {
self.confirmed_slot.store(slot_info.slot, Ordering::Relaxed);
log::debug!("confirm slot data : {slot_info:?}");
if self.confirmed_slot.load(Ordering::Relaxed) < slot_info.slot {
self.confirmed_slot.store(slot_info.slot, Ordering::Relaxed);
self.processed_storage.update_slot_info(slot_info).await;
}
self.processed_storage
.get_processed_for_slot(slot_info, &self.mints_by_index)
.await
}
Commitment::Finalized => {
self.finalized_slot.store(slot_info.slot, Ordering::Relaxed);
log::debug!("finalize slot data : {slot_info:?}");
if self.finalized_slot.load(Ordering::Relaxed) < slot_info.slot {
self.processed_storage.update_slot_info(slot_info).await;
self.finalized_slot.store(slot_info.slot, Ordering::Relaxed);
}
// move data from processed storage to main storage
let finalized_accounts = self.processed_storage.finalize(slot_info.slot).await;
let accounts_notifications = finalized_accounts
.iter()
.filter_map(|acc| {
.filter_map(|(acc, slot)| {
token_program_account_to_solana_account(
&acc.processed_account,
slot_info.slot,
*slot,
acc.write_version,
&self.mints_by_index,
)
})
.collect_vec();
for finalized_account in finalized_accounts {
for (finalized_account, _) in finalized_accounts {
self.add_account_finalized(finalized_account.processed_account)
.await;
}

View File

@ -25,6 +25,17 @@ pub enum TokenProgramAccountType {
Deleted(Pubkey),
}
impl TokenProgramAccountType {
pub fn pubkey(&self) -> Pubkey {
match self {
TokenProgramAccountType::TokenAccount(acc) => acc.pubkey,
TokenProgramAccountType::Mint(m) => m.pubkey,
TokenProgramAccountType::MultiSig(_, pk) => *pk,
TokenProgramAccountType::Deleted(pk) => *pk,
}
}
}
pub fn get_or_create_mint_index(
mint: Pubkey,
mint_index_by_pubkey: &Arc<DashMap<Pubkey, u64>>,
@ -224,7 +235,7 @@ pub fn get_token_program_account_type(
))
}
} else {
bail!("Account does not belong to token program");
unreachable!()
}
}

View File

@ -1,4 +1,11 @@
use lite_account_manager_common::account_store_interface::AccountStorageInterface;
use std::sync::Arc;
use lite_account_manager_common::{
account_data::{Account, AccountData},
account_store_interface::AccountStorageInterface,
commitment::Commitment,
slot_info::SlotInfo,
};
use lite_token_account_storage::inmemory_token_storage::InMemoryTokenStorage;
use solana_sdk::pubkey::Pubkey;
@ -6,9 +13,10 @@ mod utils;
#[tokio::test]
pub async fn test_saving_and_loading_token_account() {
tracing_subscriber::fmt::init();
let token_store = InMemoryTokenStorage::new();
let mint = Pubkey::new_unique();
let mint: Pubkey = Pubkey::new_unique();
let mint_creation_params = utils::MintCreationParams::create_default(100);
let mint_account = utils::create_mint_account_data(mint, mint_creation_params, 1, 1);
@ -22,4 +30,514 @@ pub async fn test_saving_and_loading_token_account() {
token_store
.initilize_or_update_account(token_account_data)
.await;
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Confirmed
)
.await
.unwrap()
.unwrap()
),
token_account_params
);
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Processed
)
.await
.unwrap()
.unwrap()
),
token_account_params
);
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Finalized
)
.await
.unwrap()
.unwrap()
),
token_account_params
);
assert_eq!(
utils::parse_account_data_to_mint_params(
token_store
.get_account(
mint,
lite_account_manager_common::commitment::Commitment::Confirmed
)
.await
.unwrap()
.unwrap()
),
mint_creation_params
);
assert_eq!(
utils::parse_account_data_to_mint_params(
token_store
.get_account(
mint,
lite_account_manager_common::commitment::Commitment::Processed
)
.await
.unwrap()
.unwrap()
),
mint_creation_params
);
assert_eq!(
utils::parse_account_data_to_mint_params(
token_store
.get_account(
mint,
lite_account_manager_common::commitment::Commitment::Finalized
)
.await
.unwrap()
.unwrap()
),
mint_creation_params
);
let mut rng = rand::thread_rng();
let token_account_params_2 =
utils::TokenAccountParams::create_random(&mut rng, owner, mint, 50);
let token_account_params_3 =
utils::TokenAccountParams::create_random(&mut rng, owner, mint, 100);
let token_account_data_2 =
utils::create_token_account_data(token_account_pk, token_account_params_2, 3, 3);
let account_data_3 =
utils::create_token_account_data(token_account_pk, token_account_params_3, 4, 4);
token_store
.update_account(
token_account_data_2.clone(),
lite_account_manager_common::commitment::Commitment::Processed,
)
.await;
token_store
.update_account(
account_data_3.clone(),
lite_account_manager_common::commitment::Commitment::Processed,
)
.await;
assert_eq!(
token_store
.process_slot_data(
SlotInfo {
slot: 3,
parent: 2,
root: 0,
},
Commitment::Processed
)
.await,
vec![]
);
assert_eq!(
token_store
.process_slot_data(
SlotInfo {
slot: 4,
parent: 3,
root: 0,
},
Commitment::Processed
)
.await,
vec![]
);
assert_eq!(
token_store
.process_slot_data(
SlotInfo {
slot: 3,
parent: 2,
root: 0,
},
Commitment::Confirmed
)
.await,
vec![token_account_data_2.clone()]
);
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Processed
)
.await
.unwrap()
.unwrap()
),
token_account_params_3
);
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Confirmed
)
.await
.unwrap()
.unwrap()
),
token_account_params_2
);
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Finalized
)
.await
.unwrap()
.unwrap()
),
token_account_params
);
assert_eq!(
utils::parse_account_data_to_mint_params(
token_store
.get_account(
mint,
lite_account_manager_common::commitment::Commitment::Confirmed
)
.await
.unwrap()
.unwrap()
),
mint_creation_params
);
assert_eq!(
utils::parse_account_data_to_mint_params(
token_store
.get_account(
mint,
lite_account_manager_common::commitment::Commitment::Processed
)
.await
.unwrap()
.unwrap()
),
mint_creation_params
);
assert_eq!(
utils::parse_account_data_to_mint_params(
token_store
.get_account(
mint,
lite_account_manager_common::commitment::Commitment::Finalized
)
.await
.unwrap()
.unwrap()
),
mint_creation_params
);
assert_eq!(
token_store
.process_slot_data(
SlotInfo {
slot: 4,
parent: 3,
root: 0,
},
Commitment::Confirmed
)
.await,
vec![account_data_3.clone()]
);
assert_eq!(
token_store
.process_slot_data(
SlotInfo {
slot: 3,
parent: 2,
root: 0,
},
Commitment::Finalized
)
.await,
vec![token_account_data_2.clone()]
);
let mint_2 = utils::MintCreationParams::create_random(&mut rng, 2000);
let mint_account_2 = utils::create_mint_account_data(mint, mint_2, 5, 5);
token_store
.update_account(
mint_account_2.clone(),
lite_account_manager_common::commitment::Commitment::Processed,
)
.await;
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Processed
)
.await
.unwrap()
.unwrap()
),
token_account_params_3
);
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Confirmed
)
.await
.unwrap()
.unwrap()
),
token_account_params_3
);
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Finalized
)
.await
.unwrap()
.unwrap()
),
token_account_params_2
);
assert_eq!(
utils::parse_account_data_to_mint_params(
token_store
.get_account(
mint,
lite_account_manager_common::commitment::Commitment::Confirmed
)
.await
.unwrap()
.unwrap()
),
mint_creation_params
);
assert_eq!(
utils::parse_account_data_to_mint_params(
token_store
.get_account(
mint,
lite_account_manager_common::commitment::Commitment::Processed
)
.await
.unwrap()
.unwrap()
),
mint_2
);
assert_eq!(
utils::parse_account_data_to_mint_params(
token_store
.get_account(
mint,
lite_account_manager_common::commitment::Commitment::Finalized
)
.await
.unwrap()
.unwrap()
),
mint_creation_params
);
// deletion
let deleted_account = AccountData {
pubkey: token_account_pk,
account: Arc::new(Account {
lamports: 0,
data: lite_account_manager_common::account_data::Data::Uncompressed(vec![]),
owner: Pubkey::default(),
executable: false,
rent_epoch: u64::MAX,
data_length: 0,
}),
updated_slot: 6,
write_version: 6,
};
token_store
.update_account(deleted_account.clone(), Commitment::Processed)
.await;
assert_eq!(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Processed
)
.await
.unwrap(),
None
);
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Confirmed
)
.await
.unwrap()
.unwrap()
),
token_account_params_3
);
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Finalized
)
.await
.unwrap()
.unwrap()
),
token_account_params_2
);
assert_eq!(
token_store
.process_slot_data(
SlotInfo {
slot: 6,
parent: 5,
root: 0,
},
Commitment::Confirmed
)
.await,
vec![]
);
let accounts_updated = token_store
.process_slot_data(
SlotInfo {
slot: 5,
parent: 4,
root: 0,
},
Commitment::Finalized,
)
.await;
assert!(
accounts_updated == vec![mint_account_2.clone(), account_data_3.clone()]
|| accounts_updated == vec![account_data_3.clone(), mint_account_2.clone()]
);
assert_eq!(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Processed
)
.await
.unwrap(),
None
);
assert_eq!(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Confirmed
)
.await
.unwrap(),
None
);
assert_eq!(
utils::parse_account_data_to_token_params(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Finalized
)
.await
.unwrap()
.unwrap()
),
token_account_params_3
);
assert_eq!(
token_store
.process_slot_data(
SlotInfo {
slot: 6,
parent: 5,
root: 0,
},
Commitment::Finalized
)
.await,
vec![]
);
assert_eq!(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Processed
)
.await
.unwrap(),
None
);
assert_eq!(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Confirmed
)
.await
.unwrap(),
None
);
assert_eq!(
token_store
.get_account(
token_account_pk,
lite_account_manager_common::commitment::Commitment::Finalized
)
.await
.unwrap(),
None
);
}

View File

@ -35,7 +35,7 @@ impl TokenAccountParams {
amount: u64,
) -> TokenAccountParams {
let delegate_supply: u64 = rng.gen();
let state = rng.gen::<u8>() % 3;
let state = rng.gen::<u8>() % 2;
TokenAccountParams {
owner,
mint,
@ -45,8 +45,7 @@ impl TokenAccountParams {
close_authority: Some(Pubkey::new_unique()),
state: match state {
0 => TokenProgramAccountState::Initialized,
1 => TokenProgramAccountState::Uninitialized,
2 => TokenProgramAccountState::Frozen,
1 => TokenProgramAccountState::Frozen,
_ => unreachable!(),
},
}
@ -78,7 +77,7 @@ impl MintCreationParams {
mint_authority: Some(Pubkey::new_unique()),
supply,
decimals: rng.gen(),
is_initialized: rng.gen(),
is_initialized: true,
freeze_authority: Some(Pubkey::new_unique()),
}
}