diff --git a/Cargo.lock b/Cargo.lock index 05a540351..87514299b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2635,6 +2635,7 @@ dependencies = [ "solana-sdk 0.16.0", "solana-stake-api 0.16.0", "solana-stake-program 0.16.0", + "solana-storage-api 0.16.0", "solana-vote-api 0.16.0", "solana-vote-program 0.16.0", "sys-info 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2698,7 +2699,6 @@ dependencies = [ "serde 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.92 (registry+https://github.com/rust-lang/crates.io-index)", "solana-logger 0.16.0", - "solana-runtime 0.16.0", "solana-sdk 0.16.0", ] @@ -2706,8 +2706,11 @@ dependencies = [ name = "solana-storage-program" version = "0.16.0" dependencies = [ + "assert_matches 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bincode 1.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "solana-logger 0.16.0", + "solana-runtime 0.16.0", "solana-sdk 0.16.0", "solana-storage-api 0.16.0", ] diff --git a/programs/storage_api/Cargo.toml b/programs/storage_api/Cargo.toml index 8665e19a9..b631167b3 100644 --- a/programs/storage_api/Cargo.toml +++ b/programs/storage_api/Cargo.toml @@ -17,9 +17,6 @@ serde_derive = "1.0.92" solana-logger = { path = "../../logger", version = "0.16.0" } solana-sdk = { path = "../../sdk", version = "0.16.0" } -[dev-dependencies] -solana-runtime = { path = "../../runtime", version = "0.16.0" } - [lib] crate-type = ["lib"] name = "solana_storage_api" diff --git a/programs/storage_api/src/storage_processor.rs b/programs/storage_api/src/storage_processor.rs index f7194c0ee..c7856cd8b 100644 --- a/programs/storage_api/src/storage_processor.rs +++ b/programs/storage_api/src/storage_processor.rs @@ -92,584 +92,3 @@ pub fn process_instruction( } } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::storage_contract::{ - CheckedProof, Proof, ProofStatus, StorageContract, STORAGE_ACCOUNT_SPACE, - TOTAL_REPLICATOR_REWARDS, TOTAL_VALIDATOR_REWARDS, - }; - use crate::storage_instruction; - use crate::SLOTS_PER_SEGMENT; - use crate::{get_segment_from_slot, id}; - use assert_matches::assert_matches; - use bincode::deserialize; - use log::*; - use solana_runtime::bank::Bank; - use solana_runtime::bank_client::BankClient; - use solana_sdk::account::{create_keyed_accounts, Account}; - use solana_sdk::account_utils::State; - use solana_sdk::client::SyncClient; - use solana_sdk::genesis_block::create_genesis_block; - use solana_sdk::hash::{hash, Hash}; - use solana_sdk::instruction::Instruction; - use solana_sdk::message::Message; - use solana_sdk::pubkey::Pubkey; - use solana_sdk::signature::{Keypair, KeypairUtil, Signature}; - use solana_sdk::syscall::tick_height; - use std::collections::HashMap; - use std::sync::Arc; - - const TICKS_IN_SEGMENT: u64 = SLOTS_PER_SEGMENT * DEFAULT_TICKS_PER_SLOT; - - fn test_instruction( - ix: &Instruction, - program_accounts: &mut [Account], - ) -> Result<(), InstructionError> { - let mut keyed_accounts: Vec<_> = ix - .accounts - .iter() - .zip(program_accounts.iter_mut()) - .map(|(account_meta, account)| { - KeyedAccount::new(&account_meta.pubkey, account_meta.is_signer, account) - }) - .collect(); - - let ret = process_instruction(&id(), &mut keyed_accounts, &ix.data); - info!("ret: {:?}", ret); - ret - } - - #[test] - fn test_account_owner() { - let account_owner = Pubkey::new_rand(); - let validator_storage_pubkey = Pubkey::new_rand(); - let replicator_storage_pubkey = Pubkey::new_rand(); - - let (genesis_block, mint_keypair) = create_genesis_block(1000); - let mut bank = Bank::new(&genesis_block); - let mint_pubkey = mint_keypair.pubkey(); - bank.add_instruction_processor(id(), process_instruction); - let bank = Arc::new(bank); - let bank_client = BankClient::new_shared(&bank); - - let message = Message::new(storage_instruction::create_validator_storage_account( - &mint_pubkey, - &account_owner, - &validator_storage_pubkey, - 1, - )); - bank_client - .send_message(&[&mint_keypair], message) - .expect("failed to create account"); - let account = bank - .get_account(&validator_storage_pubkey) - .expect("account not found"); - let storage_contract = account.state().expect("couldn't unpack account data"); - if let StorageContract::ValidatorStorage { owner, .. } = storage_contract { - assert_eq!(owner, account_owner); - } else { - assert!(false, "wrong account type found") - } - - let message = Message::new(storage_instruction::create_replicator_storage_account( - &mint_pubkey, - &account_owner, - &replicator_storage_pubkey, - 1, - )); - bank_client - .send_message(&[&mint_keypair], message) - .expect("failed to create account"); - let account = bank - .get_account(&replicator_storage_pubkey) - .expect("account not found"); - let storage_contract = account.state().expect("couldn't unpack account data"); - if let StorageContract::ReplicatorStorage { owner, .. } = storage_contract { - assert_eq!(owner, account_owner); - } else { - assert!(false, "wrong account type found") - } - } - - #[test] - fn test_proof_bounds() { - let account_owner = Pubkey::new_rand(); - let pubkey = Pubkey::new_rand(); - let mut account = Account { - data: vec![0; STORAGE_ACCOUNT_SPACE as usize], - ..Account::default() - }; - { - let mut storage_account = StorageAccount::new(&mut account); - storage_account - .initialize_replicator_storage(account_owner) - .unwrap(); - } - - let ix = storage_instruction::mining_proof( - &pubkey, - Hash::default(), - SLOTS_PER_SEGMENT, - Signature::default(), - ); - // the proof is for slot 16, which is in segment 0, need to move the tick height into segment 2 - let ticks_till_next_segment = TICKS_IN_SEGMENT * 2; - let mut tick_account = tick_height::create_account(1); - TickHeight::to(ticks_till_next_segment, &mut tick_account); - - assert_eq!(test_instruction(&ix, &mut [account, tick_account]), Ok(())); - } - - #[test] - fn test_storage_tx() { - let pubkey = Pubkey::new_rand(); - let mut accounts = [(pubkey, Account::default())]; - let mut keyed_accounts = create_keyed_accounts(&mut accounts); - assert!(process_instruction(&id(), &mut keyed_accounts, &[]).is_err()); - } - - #[test] - fn test_serialize_overflow() { - let pubkey = Pubkey::new_rand(); - let tick_pubkey = Pubkey::new_rand(); - let mut keyed_accounts = Vec::new(); - let mut user_account = Account::default(); - let mut tick_account = tick_height::create_account(1); - keyed_accounts.push(KeyedAccount::new(&pubkey, true, &mut user_account)); - keyed_accounts.push(KeyedAccount::new(&tick_pubkey, false, &mut tick_account)); - - let ix = storage_instruction::advertise_recent_blockhash( - &pubkey, - Hash::default(), - SLOTS_PER_SEGMENT, - ); - - assert_eq!( - process_instruction(&id(), &mut keyed_accounts, &ix.data), - Err(InstructionError::InvalidAccountData) - ); - } - - #[test] - fn test_invalid_accounts_len() { - let pubkey = Pubkey::new_rand(); - let mut accounts = [Account::default()]; - - let ix = - storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default()); - // move tick height into segment 1 - let ticks_till_next_segment = TICKS_IN_SEGMENT + 1; - let mut tick_account = tick_height::create_account(1); - TickHeight::to(ticks_till_next_segment, &mut tick_account); - - assert!(test_instruction(&ix, &mut accounts).is_err()); - - let mut accounts = [Account::default(), tick_account, Account::default()]; - - assert!(test_instruction(&ix, &mut accounts).is_err()); - } - - #[test] - fn test_submit_mining_invalid_slot() { - solana_logger::setup(); - let pubkey = Pubkey::new_rand(); - let mut accounts = [Account::default(), Account::default()]; - accounts[0].data.resize(STORAGE_ACCOUNT_SPACE as usize, 0); - accounts[1].data.resize(STORAGE_ACCOUNT_SPACE as usize, 0); - - let ix = - storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default()); - - // submitting a proof for a slot in the past, so this should fail - assert!(test_instruction(&ix, &mut accounts).is_err()); - } - - #[test] - fn test_submit_mining_ok() { - solana_logger::setup(); - let account_owner = Pubkey::new_rand(); - let pubkey = Pubkey::new_rand(); - let mut account = Account::default(); - account.data.resize(STORAGE_ACCOUNT_SPACE as usize, 0); - { - let mut storage_account = StorageAccount::new(&mut account); - storage_account - .initialize_replicator_storage(account_owner) - .unwrap(); - } - - let ix = - storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default()); - // move tick height into segment 1 - let ticks_till_next_segment = TICKS_IN_SEGMENT + 1; - let mut tick_account = tick_height::create_account(1); - TickHeight::to(ticks_till_next_segment, &mut tick_account); - - assert_matches!(test_instruction(&ix, &mut [account, tick_account]), Ok(_)); - } - - #[test] - fn test_validate_mining() { - solana_logger::setup(); - let (genesis_block, mint_keypair) = create_genesis_block(1000); - let mint_pubkey = mint_keypair.pubkey(); - - let replicator_1_storage_keypair = Keypair::new(); - let replicator_1_storage_id = replicator_1_storage_keypair.pubkey(); - - let replicator_2_storage_keypair = Keypair::new(); - let replicator_2_storage_id = replicator_2_storage_keypair.pubkey(); - - let validator_storage_keypair = Keypair::new(); - let validator_storage_id = validator_storage_keypair.pubkey(); - - let mining_pool_keypair = Keypair::new(); - let mining_pool_pubkey = mining_pool_keypair.pubkey(); - - let mut bank = Bank::new(&genesis_block); - bank.add_instruction_processor(id(), process_instruction); - let bank = Arc::new(bank); - let slot = 0; - let bank_client = BankClient::new_shared(&bank); - - init_storage_accounts( - &bank_client, - &mint_keypair, - &[&validator_storage_id], - &[&replicator_1_storage_id, &replicator_2_storage_id], - 10, - ); - let message = Message::new(storage_instruction::create_mining_pool_account( - &mint_pubkey, - &mining_pool_pubkey, - 100, - )); - bank_client.send_message(&[&mint_keypair], message).unwrap(); - - // tick the bank up until it's moved into storage segment 2 because the next advertise is for segment 1 - let next_storage_segment_tick_height = TICKS_IN_SEGMENT * 2; - for _ in 0..next_storage_segment_tick_height { - bank.register_tick(&bank.last_blockhash()); - } - - // advertise for storage segment 1 - let message = Message::new_with_payer( - vec![storage_instruction::advertise_recent_blockhash( - &validator_storage_id, - Hash::default(), - SLOTS_PER_SEGMENT, - )], - Some(&mint_pubkey), - ); - assert_matches!( - bank_client.send_message(&[&mint_keypair, &validator_storage_keypair], message), - Ok(_) - ); - - // submit proofs 5 proofs for each replicator for segment 0 - let mut checked_proofs: HashMap<_, Vec<_>> = HashMap::new(); - for slot in 0..5 { - checked_proofs - .entry(replicator_1_storage_id) - .or_default() - .push(submit_proof( - &mint_keypair, - &replicator_1_storage_keypair, - slot, - &bank_client, - )); - checked_proofs - .entry(replicator_2_storage_id) - .or_default() - .push(submit_proof( - &mint_keypair, - &replicator_2_storage_keypair, - slot, - &bank_client, - )); - } - let message = Message::new_with_payer( - vec![storage_instruction::advertise_recent_blockhash( - &validator_storage_id, - Hash::default(), - SLOTS_PER_SEGMENT * 2, - )], - Some(&mint_pubkey), - ); - - let next_storage_segment_tick_height = TICKS_IN_SEGMENT; - for _ in 0..next_storage_segment_tick_height { - bank.register_tick(&bank.last_blockhash()); - } - - assert_matches!( - bank_client.send_message(&[&mint_keypair, &validator_storage_keypair], message), - Ok(_) - ); - - let message = Message::new_with_payer( - vec![storage_instruction::proof_validation( - &validator_storage_id, - get_segment_from_slot(slot) as u64, - checked_proofs, - )], - Some(&mint_pubkey), - ); - - assert_matches!( - bank_client.send_message(&[&mint_keypair, &validator_storage_keypair], message), - Ok(_) - ); - - let message = Message::new_with_payer( - vec![storage_instruction::advertise_recent_blockhash( - &validator_storage_id, - Hash::default(), - SLOTS_PER_SEGMENT * 3, - )], - Some(&mint_pubkey), - ); - - let next_storage_segment_tick_height = TICKS_IN_SEGMENT; - for _ in 0..next_storage_segment_tick_height { - bank.register_tick(&bank.last_blockhash()); - } - - assert_matches!( - bank_client.send_message(&[&mint_keypair, &validator_storage_keypair], message), - Ok(_) - ); - - assert_eq!(bank_client.get_balance(&validator_storage_id).unwrap(), 10); - - let message = Message::new_with_payer( - vec![storage_instruction::claim_reward( - &validator_storage_id, - &mining_pool_pubkey, - slot, - )], - Some(&mint_pubkey), - ); - assert_matches!(bank_client.send_message(&[&mint_keypair], message), Ok(_)); - assert_eq!( - bank_client.get_balance(&validator_storage_id).unwrap(), - 10 + (TOTAL_VALIDATOR_REWARDS * 10) - ); - - // tick the bank into the next storage epoch so that rewards can be claimed - for _ in 0..=TICKS_IN_SEGMENT { - bank.register_tick(&bank.last_blockhash()); - } - - assert_eq!( - bank_client.get_balance(&replicator_1_storage_id).unwrap(), - 10 - ); - - let message = Message::new_with_payer( - vec![storage_instruction::claim_reward( - &replicator_1_storage_id, - &mining_pool_pubkey, - slot, - )], - Some(&mint_pubkey), - ); - assert_matches!(bank_client.send_message(&[&mint_keypair], message), Ok(_)); - - let message = Message::new_with_payer( - vec![storage_instruction::claim_reward( - &replicator_2_storage_id, - &mining_pool_pubkey, - slot, - )], - Some(&mint_pubkey), - ); - assert_matches!(bank_client.send_message(&[&mint_keypair], message), Ok(_)); - - // TODO enable when rewards are working - assert_eq!( - bank_client.get_balance(&replicator_1_storage_id).unwrap(), - 10 + (TOTAL_REPLICATOR_REWARDS * 5) - ); - } - - fn init_storage_accounts( - client: &BankClient, - mint: &Keypair, - validator_accounts_to_create: &[&Pubkey], - replicator_accounts_to_create: &[&Pubkey], - lamports: u64, - ) { - let mut ixs: Vec<_> = validator_accounts_to_create - .into_iter() - .flat_map(|account| { - storage_instruction::create_validator_storage_account( - &mint.pubkey(), - &Pubkey::default(), - account, - lamports, - ) - }) - .collect(); - replicator_accounts_to_create - .into_iter() - .for_each(|account| { - ixs.append(&mut storage_instruction::create_replicator_storage_account( - &mint.pubkey(), - &Pubkey::default(), - account, - lamports, - )) - }); - let message = Message::new(ixs); - client.send_message(&[mint], message).unwrap(); - } - - fn get_storage_slot(client: &C, account: &Pubkey) -> u64 { - match client.get_account_data(&account).unwrap() { - Some(storage_system_account_data) => { - let contract = deserialize(&storage_system_account_data); - if let Ok(contract) = contract { - match contract { - StorageContract::ValidatorStorage { slot, .. } => { - return slot; - } - _ => info!("error in reading slot"), - } - } - } - None => { - info!("error in reading slot"); - } - } - 0 - } - - fn submit_proof( - mint_keypair: &Keypair, - storage_keypair: &Keypair, - slot: u64, - bank_client: &BankClient, - ) -> CheckedProof { - let sha_state = Hash::new(Pubkey::new_rand().as_ref()); - let message = Message::new_with_payer( - vec![storage_instruction::mining_proof( - &storage_keypair.pubkey(), - sha_state, - slot, - Signature::default(), - )], - Some(&mint_keypair.pubkey()), - ); - - assert_matches!( - bank_client.send_message(&[&mint_keypair, &storage_keypair], message), - Ok(_) - ); - CheckedProof { - proof: Proof { - signature: Signature::default(), - sha_state, - }, - status: ProofStatus::Valid, - } - } - - fn get_storage_blockhash(client: &C, account: &Pubkey) -> Hash { - if let Some(storage_system_account_data) = client.get_account_data(&account).unwrap() { - let contract = deserialize(&storage_system_account_data); - if let Ok(contract) = contract { - match contract { - StorageContract::ValidatorStorage { hash, .. } => { - return hash; - } - _ => (), - } - } - } - Hash::default() - } - - #[test] - fn test_bank_storage() { - let (genesis_block, mint_keypair) = create_genesis_block(1000); - let mint_pubkey = mint_keypair.pubkey(); - let replicator_keypair = Keypair::new(); - let replicator_pubkey = replicator_keypair.pubkey(); - let validator_keypair = Keypair::new(); - let validator_pubkey = validator_keypair.pubkey(); - - let mut bank = Bank::new(&genesis_block); - bank.add_instruction_processor(id(), process_instruction); - // tick the bank up until it's moved into storage segment 2 - let next_storage_segment_tick_height = TICKS_IN_SEGMENT * 2; - for _ in 0..next_storage_segment_tick_height { - bank.register_tick(&bank.last_blockhash()); - } - let bank_client = BankClient::new(bank); - - let x = 42; - let x2 = x * 2; - let storage_blockhash = hash(&[x2]); - - bank_client - .transfer(10, &mint_keypair, &replicator_pubkey) - .unwrap(); - - let message = Message::new(storage_instruction::create_replicator_storage_account( - &mint_pubkey, - &Pubkey::default(), - &replicator_pubkey, - 1, - )); - bank_client.send_message(&[&mint_keypair], message).unwrap(); - - let message = Message::new(storage_instruction::create_validator_storage_account( - &mint_pubkey, - &Pubkey::default(), - &validator_pubkey, - 1, - )); - bank_client.send_message(&[&mint_keypair], message).unwrap(); - - let message = Message::new_with_payer( - vec![storage_instruction::advertise_recent_blockhash( - &validator_pubkey, - storage_blockhash, - SLOTS_PER_SEGMENT, - )], - Some(&mint_pubkey), - ); - - assert_matches!( - bank_client.send_message(&[&mint_keypair, &validator_keypair], message), - Ok(_) - ); - - let slot = 0; - let message = Message::new_with_payer( - vec![storage_instruction::mining_proof( - &replicator_pubkey, - Hash::default(), - slot, - Signature::default(), - )], - Some(&mint_pubkey), - ); - assert_matches!( - bank_client.send_message(&[&mint_keypair, &replicator_keypair], message), - Ok(_) - ); - - assert_eq!( - get_storage_slot(&bank_client, &validator_pubkey), - SLOTS_PER_SEGMENT - ); - assert_eq!( - get_storage_blockhash(&bank_client, &validator_pubkey), - storage_blockhash - ); - } -} diff --git a/programs/storage_program/Cargo.toml b/programs/storage_program/Cargo.toml index 40e0f1481..748b4bc34 100644 --- a/programs/storage_program/Cargo.toml +++ b/programs/storage_program/Cargo.toml @@ -14,6 +14,11 @@ solana-logger = { path = "../../logger", version = "0.16.0" } solana-sdk = { path = "../../sdk", version = "0.16.0" } solana-storage-api = { path = "../storage_api", version = "0.16.0" } +[dev-dependencies] +solana-runtime = { path = "../../runtime", version = "0.16.0" } +assert_matches = "1.3.0" +bincode = "1.1.4" + [lib] crate-type = ["lib", "cdylib"] name = "solana_storage_program" diff --git a/programs/storage_program/tests/storage_processor.rs b/programs/storage_program/tests/storage_processor.rs new file mode 100644 index 000000000..c6f31356d --- /dev/null +++ b/programs/storage_program/tests/storage_processor.rs @@ -0,0 +1,577 @@ +use assert_matches::assert_matches; +use bincode::deserialize; +use log::*; +use solana_runtime::bank::Bank; +use solana_runtime::bank_client::BankClient; +use solana_sdk::account::{create_keyed_accounts, Account, KeyedAccount}; +use solana_sdk::account_utils::State; +use solana_sdk::client::SyncClient; +use solana_sdk::genesis_block::create_genesis_block; +use solana_sdk::hash::{hash, Hash}; +use solana_sdk::instruction::{Instruction, InstructionError}; +use solana_sdk::message::Message; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::{Keypair, KeypairUtil, Signature}; +use solana_sdk::syscall::tick_height; +use solana_sdk::syscall::tick_height::TickHeight; +use solana_sdk::timing::DEFAULT_TICKS_PER_SLOT; +use solana_storage_api::storage_contract::StorageAccount; +use solana_storage_api::storage_contract::{ + CheckedProof, Proof, ProofStatus, StorageContract, STORAGE_ACCOUNT_SPACE, + TOTAL_REPLICATOR_REWARDS, TOTAL_VALIDATOR_REWARDS, +}; +use solana_storage_api::storage_instruction; +use solana_storage_api::storage_processor::process_instruction; +use solana_storage_api::SLOTS_PER_SEGMENT; +use solana_storage_api::{get_segment_from_slot, id}; +use std::collections::HashMap; +use std::sync::Arc; + +const TICKS_IN_SEGMENT: u64 = SLOTS_PER_SEGMENT * DEFAULT_TICKS_PER_SLOT; + +fn test_instruction( + ix: &Instruction, + program_accounts: &mut [Account], +) -> Result<(), InstructionError> { + let mut keyed_accounts: Vec<_> = ix + .accounts + .iter() + .zip(program_accounts.iter_mut()) + .map(|(account_meta, account)| { + KeyedAccount::new(&account_meta.pubkey, account_meta.is_signer, account) + }) + .collect(); + + let ret = process_instruction(&id(), &mut keyed_accounts, &ix.data); + info!("ret: {:?}", ret); + ret +} + +#[test] +fn test_account_owner() { + let account_owner = Pubkey::new_rand(); + let validator_storage_pubkey = Pubkey::new_rand(); + let replicator_storage_pubkey = Pubkey::new_rand(); + + let (genesis_block, mint_keypair) = create_genesis_block(1000); + let mut bank = Bank::new(&genesis_block); + let mint_pubkey = mint_keypair.pubkey(); + bank.add_instruction_processor(id(), process_instruction); + let bank = Arc::new(bank); + let bank_client = BankClient::new_shared(&bank); + + let message = Message::new(storage_instruction::create_validator_storage_account( + &mint_pubkey, + &account_owner, + &validator_storage_pubkey, + 1, + )); + bank_client + .send_message(&[&mint_keypair], message) + .expect("failed to create account"); + let account = bank + .get_account(&validator_storage_pubkey) + .expect("account not found"); + let storage_contract = account.state().expect("couldn't unpack account data"); + if let StorageContract::ValidatorStorage { owner, .. } = storage_contract { + assert_eq!(owner, account_owner); + } else { + assert!(false, "wrong account type found") + } + + let message = Message::new(storage_instruction::create_replicator_storage_account( + &mint_pubkey, + &account_owner, + &replicator_storage_pubkey, + 1, + )); + bank_client + .send_message(&[&mint_keypair], message) + .expect("failed to create account"); + let account = bank + .get_account(&replicator_storage_pubkey) + .expect("account not found"); + let storage_contract = account.state().expect("couldn't unpack account data"); + if let StorageContract::ReplicatorStorage { owner, .. } = storage_contract { + assert_eq!(owner, account_owner); + } else { + assert!(false, "wrong account type found") + } +} + +#[test] +fn test_proof_bounds() { + let account_owner = Pubkey::new_rand(); + let pubkey = Pubkey::new_rand(); + let mut account = Account { + data: vec![0; STORAGE_ACCOUNT_SPACE as usize], + ..Account::default() + }; + { + let mut storage_account = StorageAccount::new(&mut account); + storage_account + .initialize_replicator_storage(account_owner) + .unwrap(); + } + + let ix = storage_instruction::mining_proof( + &pubkey, + Hash::default(), + SLOTS_PER_SEGMENT, + Signature::default(), + ); + // the proof is for slot 16, which is in segment 0, need to move the tick height into segment 2 + let ticks_till_next_segment = TICKS_IN_SEGMENT * 2; + let mut tick_account = tick_height::create_account(1); + TickHeight::to(ticks_till_next_segment, &mut tick_account); + + assert_eq!(test_instruction(&ix, &mut [account, tick_account]), Ok(())); +} + +#[test] +fn test_storage_tx() { + let pubkey = Pubkey::new_rand(); + let mut accounts = [(pubkey, Account::default())]; + let mut keyed_accounts = create_keyed_accounts(&mut accounts); + assert!(process_instruction(&id(), &mut keyed_accounts, &[]).is_err()); +} + +#[test] +fn test_serialize_overflow() { + let pubkey = Pubkey::new_rand(); + let tick_pubkey = Pubkey::new_rand(); + let mut keyed_accounts = Vec::new(); + let mut user_account = Account::default(); + let mut tick_account = tick_height::create_account(1); + keyed_accounts.push(KeyedAccount::new(&pubkey, true, &mut user_account)); + keyed_accounts.push(KeyedAccount::new(&tick_pubkey, false, &mut tick_account)); + + let ix = storage_instruction::advertise_recent_blockhash( + &pubkey, + Hash::default(), + SLOTS_PER_SEGMENT, + ); + + assert_eq!( + process_instruction(&id(), &mut keyed_accounts, &ix.data), + Err(InstructionError::InvalidAccountData) + ); +} + +#[test] +fn test_invalid_accounts_len() { + let pubkey = Pubkey::new_rand(); + let mut accounts = [Account::default()]; + + let ix = storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default()); + // move tick height into segment 1 + let ticks_till_next_segment = TICKS_IN_SEGMENT + 1; + let mut tick_account = tick_height::create_account(1); + TickHeight::to(ticks_till_next_segment, &mut tick_account); + + assert!(test_instruction(&ix, &mut accounts).is_err()); + + let mut accounts = [Account::default(), tick_account, Account::default()]; + + assert!(test_instruction(&ix, &mut accounts).is_err()); +} + +#[test] +fn test_submit_mining_invalid_slot() { + solana_logger::setup(); + let pubkey = Pubkey::new_rand(); + let mut accounts = [Account::default(), Account::default()]; + accounts[0].data.resize(STORAGE_ACCOUNT_SPACE as usize, 0); + accounts[1].data.resize(STORAGE_ACCOUNT_SPACE as usize, 0); + + let ix = storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default()); + + // submitting a proof for a slot in the past, so this should fail + assert!(test_instruction(&ix, &mut accounts).is_err()); +} + +#[test] +fn test_submit_mining_ok() { + solana_logger::setup(); + let account_owner = Pubkey::new_rand(); + let pubkey = Pubkey::new_rand(); + let mut account = Account::default(); + account.data.resize(STORAGE_ACCOUNT_SPACE as usize, 0); + { + let mut storage_account = StorageAccount::new(&mut account); + storage_account + .initialize_replicator_storage(account_owner) + .unwrap(); + } + + let ix = storage_instruction::mining_proof(&pubkey, Hash::default(), 0, Signature::default()); + // move tick height into segment 1 + let ticks_till_next_segment = TICKS_IN_SEGMENT + 1; + let mut tick_account = tick_height::create_account(1); + TickHeight::to(ticks_till_next_segment, &mut tick_account); + + assert_matches!(test_instruction(&ix, &mut [account, tick_account]), Ok(_)); +} + +#[test] +fn test_validate_mining() { + solana_logger::setup(); + let (genesis_block, mint_keypair) = create_genesis_block(1000); + let mint_pubkey = mint_keypair.pubkey(); + + let replicator_1_storage_keypair = Keypair::new(); + let replicator_1_storage_id = replicator_1_storage_keypair.pubkey(); + + let replicator_2_storage_keypair = Keypair::new(); + let replicator_2_storage_id = replicator_2_storage_keypair.pubkey(); + + let validator_storage_keypair = Keypair::new(); + let validator_storage_id = validator_storage_keypair.pubkey(); + + let mining_pool_keypair = Keypair::new(); + let mining_pool_pubkey = mining_pool_keypair.pubkey(); + + let mut bank = Bank::new(&genesis_block); + bank.add_instruction_processor(id(), process_instruction); + let bank = Arc::new(bank); + let slot = 0; + let bank_client = BankClient::new_shared(&bank); + + init_storage_accounts( + &bank_client, + &mint_keypair, + &[&validator_storage_id], + &[&replicator_1_storage_id, &replicator_2_storage_id], + 10, + ); + let message = Message::new(storage_instruction::create_mining_pool_account( + &mint_pubkey, + &mining_pool_pubkey, + 100, + )); + bank_client.send_message(&[&mint_keypair], message).unwrap(); + + // tick the bank up until it's moved into storage segment 2 because the next advertise is for segment 1 + let next_storage_segment_tick_height = TICKS_IN_SEGMENT * 2; + for _ in 0..next_storage_segment_tick_height { + bank.register_tick(&bank.last_blockhash()); + } + + // advertise for storage segment 1 + let message = Message::new_with_payer( + vec![storage_instruction::advertise_recent_blockhash( + &validator_storage_id, + Hash::default(), + SLOTS_PER_SEGMENT, + )], + Some(&mint_pubkey), + ); + assert_matches!( + bank_client.send_message(&[&mint_keypair, &validator_storage_keypair], message), + Ok(_) + ); + + // submit proofs 5 proofs for each replicator for segment 0 + let mut checked_proofs: HashMap<_, Vec<_>> = HashMap::new(); + for slot in 0..5 { + checked_proofs + .entry(replicator_1_storage_id) + .or_default() + .push(submit_proof( + &mint_keypair, + &replicator_1_storage_keypair, + slot, + &bank_client, + )); + checked_proofs + .entry(replicator_2_storage_id) + .or_default() + .push(submit_proof( + &mint_keypair, + &replicator_2_storage_keypair, + slot, + &bank_client, + )); + } + let message = Message::new_with_payer( + vec![storage_instruction::advertise_recent_blockhash( + &validator_storage_id, + Hash::default(), + SLOTS_PER_SEGMENT * 2, + )], + Some(&mint_pubkey), + ); + + let next_storage_segment_tick_height = TICKS_IN_SEGMENT; + for _ in 0..next_storage_segment_tick_height { + bank.register_tick(&bank.last_blockhash()); + } + + assert_matches!( + bank_client.send_message(&[&mint_keypair, &validator_storage_keypair], message), + Ok(_) + ); + + let message = Message::new_with_payer( + vec![storage_instruction::proof_validation( + &validator_storage_id, + get_segment_from_slot(slot) as u64, + checked_proofs, + )], + Some(&mint_pubkey), + ); + + assert_matches!( + bank_client.send_message(&[&mint_keypair, &validator_storage_keypair], message), + Ok(_) + ); + + let message = Message::new_with_payer( + vec![storage_instruction::advertise_recent_blockhash( + &validator_storage_id, + Hash::default(), + SLOTS_PER_SEGMENT * 3, + )], + Some(&mint_pubkey), + ); + + let next_storage_segment_tick_height = TICKS_IN_SEGMENT; + for _ in 0..next_storage_segment_tick_height { + bank.register_tick(&bank.last_blockhash()); + } + + assert_matches!( + bank_client.send_message(&[&mint_keypair, &validator_storage_keypair], message), + Ok(_) + ); + + assert_eq!(bank_client.get_balance(&validator_storage_id).unwrap(), 10); + + let message = Message::new_with_payer( + vec![storage_instruction::claim_reward( + &validator_storage_id, + &mining_pool_pubkey, + slot, + )], + Some(&mint_pubkey), + ); + assert_matches!(bank_client.send_message(&[&mint_keypair], message), Ok(_)); + assert_eq!( + bank_client.get_balance(&validator_storage_id).unwrap(), + 10 + (TOTAL_VALIDATOR_REWARDS * 10) + ); + + // tick the bank into the next storage epoch so that rewards can be claimed + for _ in 0..=TICKS_IN_SEGMENT { + bank.register_tick(&bank.last_blockhash()); + } + + assert_eq!( + bank_client.get_balance(&replicator_1_storage_id).unwrap(), + 10 + ); + + let message = Message::new_with_payer( + vec![storage_instruction::claim_reward( + &replicator_1_storage_id, + &mining_pool_pubkey, + slot, + )], + Some(&mint_pubkey), + ); + assert_matches!(bank_client.send_message(&[&mint_keypair], message), Ok(_)); + + let message = Message::new_with_payer( + vec![storage_instruction::claim_reward( + &replicator_2_storage_id, + &mining_pool_pubkey, + slot, + )], + Some(&mint_pubkey), + ); + assert_matches!(bank_client.send_message(&[&mint_keypair], message), Ok(_)); + + // TODO enable when rewards are working + assert_eq!( + bank_client.get_balance(&replicator_1_storage_id).unwrap(), + 10 + (TOTAL_REPLICATOR_REWARDS * 5) + ); +} + +fn init_storage_accounts( + client: &BankClient, + mint: &Keypair, + validator_accounts_to_create: &[&Pubkey], + replicator_accounts_to_create: &[&Pubkey], + lamports: u64, +) { + let mut ixs: Vec<_> = validator_accounts_to_create + .into_iter() + .flat_map(|account| { + storage_instruction::create_validator_storage_account( + &mint.pubkey(), + &Pubkey::default(), + account, + lamports, + ) + }) + .collect(); + replicator_accounts_to_create + .into_iter() + .for_each(|account| { + ixs.append(&mut storage_instruction::create_replicator_storage_account( + &mint.pubkey(), + &Pubkey::default(), + account, + lamports, + )) + }); + let message = Message::new(ixs); + client.send_message(&[mint], message).unwrap(); +} + +fn get_storage_slot(client: &C, account: &Pubkey) -> u64 { + match client.get_account_data(&account).unwrap() { + Some(storage_system_account_data) => { + let contract = deserialize(&storage_system_account_data); + if let Ok(contract) = contract { + match contract { + StorageContract::ValidatorStorage { slot, .. } => { + return slot; + } + _ => info!("error in reading slot"), + } + } + } + None => { + info!("error in reading slot"); + } + } + 0 +} + +fn submit_proof( + mint_keypair: &Keypair, + storage_keypair: &Keypair, + slot: u64, + bank_client: &BankClient, +) -> CheckedProof { + let sha_state = Hash::new(Pubkey::new_rand().as_ref()); + let message = Message::new_with_payer( + vec![storage_instruction::mining_proof( + &storage_keypair.pubkey(), + sha_state, + slot, + Signature::default(), + )], + Some(&mint_keypair.pubkey()), + ); + + assert_matches!( + bank_client.send_message(&[&mint_keypair, &storage_keypair], message), + Ok(_) + ); + CheckedProof { + proof: Proof { + signature: Signature::default(), + sha_state, + }, + status: ProofStatus::Valid, + } +} + +fn get_storage_blockhash(client: &C, account: &Pubkey) -> Hash { + if let Some(storage_system_account_data) = client.get_account_data(&account).unwrap() { + let contract = deserialize(&storage_system_account_data); + if let Ok(contract) = contract { + match contract { + StorageContract::ValidatorStorage { hash, .. } => { + return hash; + } + _ => (), + } + } + } + Hash::default() +} + +#[test] +fn test_bank_storage() { + let (genesis_block, mint_keypair) = create_genesis_block(1000); + let mint_pubkey = mint_keypair.pubkey(); + let replicator_keypair = Keypair::new(); + let replicator_pubkey = replicator_keypair.pubkey(); + let validator_keypair = Keypair::new(); + let validator_pubkey = validator_keypair.pubkey(); + + let mut bank = Bank::new(&genesis_block); + bank.add_instruction_processor(id(), process_instruction); + // tick the bank up until it's moved into storage segment 2 + let next_storage_segment_tick_height = TICKS_IN_SEGMENT * 2; + for _ in 0..next_storage_segment_tick_height { + bank.register_tick(&bank.last_blockhash()); + } + let bank_client = BankClient::new(bank); + + let x = 42; + let x2 = x * 2; + let storage_blockhash = hash(&[x2]); + + bank_client + .transfer(10, &mint_keypair, &replicator_pubkey) + .unwrap(); + + let message = Message::new(storage_instruction::create_replicator_storage_account( + &mint_pubkey, + &Pubkey::default(), + &replicator_pubkey, + 1, + )); + bank_client.send_message(&[&mint_keypair], message).unwrap(); + + let message = Message::new(storage_instruction::create_validator_storage_account( + &mint_pubkey, + &Pubkey::default(), + &validator_pubkey, + 1, + )); + bank_client.send_message(&[&mint_keypair], message).unwrap(); + + let message = Message::new_with_payer( + vec![storage_instruction::advertise_recent_blockhash( + &validator_pubkey, + storage_blockhash, + SLOTS_PER_SEGMENT, + )], + Some(&mint_pubkey), + ); + + assert_matches!( + bank_client.send_message(&[&mint_keypair, &validator_keypair], message), + Ok(_) + ); + + let slot = 0; + let message = Message::new_with_payer( + vec![storage_instruction::mining_proof( + &replicator_pubkey, + Hash::default(), + slot, + Signature::default(), + )], + Some(&mint_pubkey), + ); + assert_matches!( + bank_client.send_message(&[&mint_keypair, &replicator_keypair], message), + Ok(_) + ); + + assert_eq!( + get_storage_slot(&bank_client, &validator_pubkey), + SLOTS_PER_SEGMENT + ); + assert_eq!( + get_storage_blockhash(&bank_client, &validator_pubkey), + storage_blockhash + ); +} diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 4a1a154f0..70097ffdc 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -29,6 +29,7 @@ solana-noop-program = { path = "../programs/noop_program", version = "0.16.0" } solana-sdk = { path = "../sdk", version = "0.16.0" } solana-stake-api = { path = "../programs/stake_api", version = "0.16.0" } solana-stake-program = { path = "../programs/stake_program", version = "0.16.0" } +solana-storage-api = { path = "../programs/storage_api", version = "0.16.0" } solana-vote-api = { path = "../programs/vote_api", version = "0.16.0" } solana-vote-program = { path = "../programs/vote_program", version = "0.16.0" } sys-info = "0.5.7" diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index e85a9315d..adce088f5 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -14,6 +14,8 @@ use crate::serde_utils::{ }; use crate::stakes::Stakes; use crate::status_cache::StatusCache; +use crate::storage_utils; +use crate::storage_utils::StorageAccounts; use bincode::{deserialize_from, serialize, serialize_into, serialized_size}; use log::*; use serde::{Deserialize, Serialize}; @@ -232,6 +234,9 @@ pub struct Bank { /// cache of vote_account and stake_account state for this fork stakes: RwLock, + /// cache of validator and replicator storage accounts for this fork + storage_accounts: RwLock, + /// staked nodes on epoch boundaries, saved off when a bank.slot() is at /// a leader schedule calculation boundary epoch_stakes: HashMap, @@ -287,6 +292,7 @@ impl Bank { bank.transaction_count .store(parent.transaction_count() as usize, Ordering::Relaxed); bank.stakes = RwLock::new(parent.stakes.read().unwrap().clone()); + bank.storage_accounts = RwLock::new(parent.storage_accounts.read().unwrap().clone()); bank.tick_height .store(parent.tick_height.load(Ordering::SeqCst), Ordering::SeqCst); @@ -936,7 +942,7 @@ impl Bank { .accounts .store_accounts(self.slot(), txs, executed, loaded_accounts); - self.store_stakes(txs, executed, loaded_accounts); + self.update_cached_accounts(txs, executed, loaded_accounts); // once committed there is no way to unroll let write_elapsed = now.elapsed(); @@ -1005,6 +1011,11 @@ impl Bank { if Stakes::is_stake(account) { self.stakes.write().unwrap().store(pubkey, account); + } else if storage_utils::is_storage(account) { + self.storage_accounts + .write() + .unwrap() + .store(pubkey, account); } } @@ -1129,7 +1140,7 @@ impl Bank { } /// a bank-level cache of vote accounts - fn store_stakes( + fn update_cached_accounts( &self, txs: &[Transaction], res: &[Result<()>], @@ -1143,17 +1154,31 @@ impl Bank { let message = &txs[i].message(); let acc = raccs.as_ref().unwrap(); - for (pubkey, account) in message - .account_keys - .iter() - .zip(acc.0.iter()) - .filter(|(_, account)| Stakes::is_stake(account)) + for (pubkey, account) in + message + .account_keys + .iter() + .zip(acc.0.iter()) + .filter(|(_, account)| { + (Stakes::is_stake(account)) || storage_utils::is_storage(account) + }) { - self.stakes.write().unwrap().store(pubkey, account); + if Stakes::is_stake(account) { + self.stakes.write().unwrap().store(pubkey, account); + } else if storage_utils::is_storage(account) { + self.storage_accounts + .write() + .unwrap() + .store(pubkey, account); + } } } } + pub fn storage_accounts(&self) -> StorageAccounts { + self.storage_accounts.read().unwrap().clone() + } + /// current vote accounts for this bank along with the stake /// attributed to each account pub fn vote_accounts(&self) -> HashMap { diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 89d4ff0f4..44f7c64ec 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -15,6 +15,7 @@ mod native_loader; mod serde_utils; pub mod stakes; pub mod status_cache; +pub mod storage_utils; mod system_instruction_processor; #[macro_use] diff --git a/runtime/src/storage_utils.rs b/runtime/src/storage_utils.rs new file mode 100644 index 000000000..9253a66b9 --- /dev/null +++ b/runtime/src/storage_utils.rs @@ -0,0 +1,116 @@ +use crate::bank::Bank; +use solana_sdk::account::Account; +use solana_sdk::account_utils::State; +use solana_sdk::pubkey::Pubkey; +use solana_storage_api::storage_contract::StorageContract; +use std::collections::{HashMap, HashSet}; + +#[derive(Default, Clone, PartialEq, Debug, Deserialize, Serialize)] +pub struct StorageAccounts { + /// validator storage accounts + validator_accounts: HashSet, + + /// replicator storage accounts + replicator_accounts: HashSet, +} + +pub fn is_storage(account: &Account) -> bool { + solana_storage_api::check_id(&account.owner) +} + +impl StorageAccounts { + pub fn store(&mut self, pubkey: &Pubkey, account: &Account) { + if let Ok(storage_state) = account.state() { + if let StorageContract::ReplicatorStorage { .. } = storage_state { + if account.lamports == 0 { + self.replicator_accounts.remove(pubkey); + } else { + self.replicator_accounts.insert(*pubkey); + } + } else if let StorageContract::ValidatorStorage { .. } = storage_state { + if account.lamports == 0 { + self.validator_accounts.remove(pubkey); + } else { + self.validator_accounts.insert(*pubkey); + } + } + }; + } +} + +pub fn validator_accounts(bank: &Bank) -> HashMap { + bank.storage_accounts() + .validator_accounts + .iter() + .filter_map(|account_id| { + bank.get_account(account_id) + .and_then(|account| Some((*account_id, account))) + }) + .collect() +} + +pub fn replicator_accounts(bank: &Bank) -> HashMap { + bank.storage_accounts() + .replicator_accounts + .iter() + .filter_map(|account_id| { + bank.get_account(account_id) + .and_then(|account| Some((*account_id, account))) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bank_client::BankClient; + use solana_sdk::client::SyncClient; + use solana_sdk::genesis_block::create_genesis_block; + use solana_sdk::message::Message; + use solana_sdk::signature::{Keypair, KeypairUtil}; + use solana_storage_api::{storage_instruction, storage_processor}; + use std::sync::Arc; + + #[test] + fn test_store_and_recover() { + let (genesis_block, mint_keypair) = create_genesis_block(1000); + let mint_pubkey = mint_keypair.pubkey(); + let replicator_keypair = Keypair::new(); + let replicator_pubkey = replicator_keypair.pubkey(); + let validator_keypair = Keypair::new(); + let validator_pubkey = validator_keypair.pubkey(); + let mut bank = Bank::new(&genesis_block); + bank.add_instruction_processor( + solana_storage_api::id(), + storage_processor::process_instruction, + ); + + let bank = Arc::new(bank); + let bank_client = BankClient::new_shared(&bank); + + bank_client + .transfer(10, &mint_keypair, &replicator_pubkey) + .unwrap(); + let message = Message::new(storage_instruction::create_replicator_storage_account( + &mint_pubkey, + &Pubkey::default(), + &replicator_pubkey, + 1, + )); + bank_client.send_message(&[&mint_keypair], message).unwrap(); + + bank_client + .transfer(10, &mint_keypair, &validator_pubkey) + .unwrap(); + let message = Message::new(storage_instruction::create_validator_storage_account( + &mint_pubkey, + &Pubkey::default(), + &validator_pubkey, + 1, + )); + bank_client.send_message(&[&mint_keypair], message).unwrap(); + + assert_eq!(validator_accounts(bank.as_ref()).len(), 1); + assert_eq!(replicator_accounts(bank.as_ref()).len(), 1); + } +}