blockworks-foundation
/
solana
mirror of https://github.com/blockworks-foundation/solana.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Integrate program loader-v4 with bank (#32832) 

* Integrate program loader-v4 with bank

* fix tests

* new struct for ProgramRuntimeEnvironments

* remove environment from program_runtime_environment_v

* move find_program_in_cache() to invoke_context

* cleanup
This commit is contained in:
Pankaj Garg 2023-08-16 10:50:23 -07:00 committed by GitHub
parent d5d4732f17
commit c17b938204
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 205 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ledger-tool/src/program.rs
View File

@ -536,8 +536,14 @@ pub fn program(ledger_path: &Path, matches: &ArgMatches<'_>) {
with_mock_invoke_context!(invoke_context, transaction_context, transaction_accounts);
// Adding `DELAY_VISIBILITY_SLOT_OFFSET` to slots to accommodate for delay visibility of the program
let mut loaded_programs =
LoadedProgramsForTxBatch::new(bank.slot() + DELAY_VISIBILITY_SLOT_OFFSET);
let mut loaded_programs = LoadedProgramsForTxBatch::new(
bank.slot() + DELAY_VISIBILITY_SLOT_OFFSET,
bank.loaded_programs_cache
.read()
.unwrap()
.environments
.clone(),
);
for key in cached_account_keys {
loaded_programs.replenish(key, bank.load_program(&key));
debug!("Loaded program {}", key);

8
program-runtime/src/invoke_context.rs
View File

@ -214,6 +214,14 @@ impl<'a> InvokeContext<'a> {
}
}
pub fn find_program_in_cache(&self, pubkey: &Pubkey) -> Option<Arc<LoadedProgram>> {
// First lookup the cache of the programs modified by the current transaction. If not found, lookup
// the cache of the cache of the programs that are loaded for the transaction batch.
self.programs_modified_by_tx
.find(pubkey)
.or_else(|| self.programs_loaded_for_tx_batch.find(pubkey))
}
/// Push a stack frame onto the invocation stack
pub fn push(&mut self) -> Result<(), InstructionError> {
let instruction_context = self

26
program-runtime/src/loaded_programs.rs
View File

@ -347,16 +347,21 @@ impl LoadedProgram {
}
}
#[derive(Clone, Debug, Default)]
pub struct ProgramRuntimeEnvironments {
/// Globally shared RBPF config and syscall registry
pub program_runtime_v1: Arc<BuiltinProgram<InvokeContext<'static>>>,
/// Globally shared RBPF config and syscall registry for runtime V2
pub program_runtime_v2: Arc<BuiltinProgram<InvokeContext<'static>>>,
}
#[derive(Debug, Default)]
pub struct LoadedPrograms {
/// A two level index:
///
/// Pubkey is the address of a program, multiple versions can coexists simultaneously under the same address (in different slots).
entries: HashMap<Pubkey, Vec<Arc<LoadedProgram>>>,
/// Globally shared RBPF config and syscall registry
pub program_runtime_environment_v1: Arc<BuiltinProgram<InvokeContext<'static>>>,
/// Globally shared RBPF config and syscall registry for runtime V2
pub program_runtime_environment_v2: Arc<BuiltinProgram<InvokeContext<'static>>>,
pub environments: ProgramRuntimeEnvironments,
latest_root: Slot,
pub stats: Stats,
}
@ -367,13 +372,15 @@ pub struct LoadedProgramsForTxBatch {
/// LoadedProgram is the corresponding program entry valid for the slot in which a transaction is being executed.
entries: HashMap<Pubkey, Arc<LoadedProgram>>,
slot: Slot,
pub environments: ProgramRuntimeEnvironments,
}
impl LoadedProgramsForTxBatch {
pub fn new(slot: Slot) -> Self {
pub fn new(slot: Slot, environments: ProgramRuntimeEnvironments) -> Self {
Self {
entries: HashMap::new(),
slot,
environments,
}
}
@ -496,22 +503,22 @@ impl LoadedPrograms {
LoadedProgramType::LegacyV0(program) | LoadedProgramType::LegacyV1(program)
if Arc::ptr_eq(
program.get_loader(),
&self.program_runtime_environment_v1,
&self.environments.program_runtime_v1,
) =>
{
true
}
LoadedProgramType::Unloaded(environment)
| LoadedProgramType::FailedVerification(environment)
if Arc::ptr_eq(environment, &self.program_runtime_environment_v1)
|| Arc::ptr_eq(environment, &self.program_runtime_environment_v2) =>
if Arc::ptr_eq(environment, &self.environments.program_runtime_v1)
|| Arc::ptr_eq(environment, &self.environments.program_runtime_v2) =>
{
true
}
LoadedProgramType::Typed(program)
if Arc::ptr_eq(
program.get_loader(),
&self.program_runtime_environment_v2,
&self.environments.program_runtime_v2,
) =>
{
true
@ -654,6 +661,7 @@ impl LoadedPrograms {
LoadedProgramsForTxBatch {
entries: found,
slot: working_slot.current_slot(),
environments: self.environments.clone(),
},
missing,
)

17
programs/bpf_loader/src/lib.rs
View File

@ -99,18 +99,6 @@ pub fn load_program_from_bytes(
Ok(loaded_program)
}
fn find_program_in_cache(
invoke_context: &InvokeContext,
pubkey: &Pubkey,
) -> Option<Arc<LoadedProgram>> {
// First lookup the cache of the programs modified by the current transaction. If not found, lookup
// the cache of the cache of the programs that are loaded for the transaction batch.
invoke_context
.programs_modified_by_tx
.find(pubkey)
.or_else(|| invoke_context.programs_loaded_for_tx_batch.find(pubkey))
}
macro_rules! deploy_program {
($invoke_context:expr, $program_id:expr, $loader_key:expr,
$account_size:expr, $slot:expr, $drop:expr, $new_programdata:expr $(,)?) => {{
@ -137,7 +125,7 @@ macro_rules! deploy_program {
$slot,
Arc::new(program_runtime_environment),
)?;
if let Some(old_entry) = find_program_in_cache($invoke_context, &$program_id) {
if let Some(old_entry) = $invoke_context.find_program_in_cache(&$program_id) {
executor.tx_usage_counter.store(
old_entry.tx_usage_counter.load(Ordering::Relaxed),
Ordering::Relaxed
@ -507,7 +495,8 @@ fn process_instruction_inner(
}
let mut get_or_create_executor_time = Measure::start("get_or_create_executor_time");
let executor = find_program_in_cache(invoke_context, program_account.get_key())
let executor = invoke_context
.find_program_in_cache(program_account.get_key())
.ok_or(InstructionError::InvalidAccountData)?;
if executor.is_tombstone() {

166
programs/loader-v4/src/lib.rs
View File

@ -5,7 +5,9 @@ use {
compute_budget::ComputeBudget,
ic_logger_msg,
invoke_context::InvokeContext,
loaded_programs::{LoadProgramMetrics, LoadedProgram, LoadedProgramType},
loaded_programs::{
LoadProgramMetrics, LoadedProgram, LoadedProgramType, DELAY_VISIBILITY_SLOT_OFFSET,
},
log_collector::LogCollector,
stable_log,
},
@ -22,7 +24,7 @@ use {
},
solana_sdk::{
entrypoint::{HEAP_LENGTH, SUCCESS},
feature_set::{self, FeatureSet},
feature_set,
instruction::InstructionError,
loader_v4::{self, LoaderV4State, DEPLOYMENT_COOLDOWN_IN_SLOTS},
loader_v4_instruction::LoaderV4Instruction,
@ -100,42 +102,6 @@ pub fn create_program_runtime_environment_v2<'a>(
BuiltinProgram::new_loader(config)
}
pub fn load_program_from_account(
_feature_set: &FeatureSet,
compute_budget: &ComputeBudget,
log_collector: Option<Rc<RefCell<LogCollector>>>,
program: &BorrowedAccount,
debugging_features: bool,
) -> Result<(Arc<LoadedProgram>, LoadProgramMetrics), InstructionError> {
let mut load_program_metrics = LoadProgramMetrics {
program_id: program.get_key().to_string(),
..LoadProgramMetrics::default()
};
let state = get_state(program.get_data())?;
let programdata = program
.get_data()
.get(LoaderV4State::program_data_offset()..)
.ok_or(InstructionError::AccountDataTooSmall)?;
let loaded_program = LoadedProgram::new(
&loader_v4::id(),
Arc::new(create_program_runtime_environment_v2(
compute_budget,
debugging_features,
)),
state.slot,
state.slot.saturating_add(1),
None,
programdata,
program.get_data().len(),
&mut load_program_metrics,
)
.map_err(|err| {
ic_logger_msg!(log_collector, "{}", err);
InstructionError::InvalidAccountData
})?;
Ok((Arc::new(loaded_program), load_program_metrics))
}
fn calculate_heap_cost(heap_size: u64, heap_cost: u64) -> u64 {
const KIBIBYTE: u64 = 1024;
const PAGE_SIZE_KB: u64 = 32;
@ -458,13 +424,37 @@ pub fn process_instruction_deploy(
} else {
&program
};
let (_executor, load_program_metrics) = load_program_from_account(
&invoke_context.feature_set,
invoke_context.get_compute_budget(),
invoke_context.get_log_collector(),
buffer,
false, /* debugging_features */
)?;
let programdata = buffer
.get_data()
.get(LoaderV4State::program_data_offset()..)
.ok_or(InstructionError::AccountDataTooSmall)?;
let deployment_slot = state.slot;
let effective_slot = deployment_slot.saturating_add(DELAY_VISIBILITY_SLOT_OFFSET);
let mut load_program_metrics = LoadProgramMetrics {
program_id: buffer.get_key().to_string(),
..LoadProgramMetrics::default()
};
let executor = LoadedProgram::new(
&loader_v4::id(),
invoke_context
.programs_modified_by_tx
.environments
.program_runtime_v2
.clone(),
deployment_slot,
effective_slot,
None,
programdata,
buffer.get_data().len(),
&mut load_program_metrics,
)
.map_err(|err| {
ic_logger_msg!(log_collector, "{}", err);
InstructionError::InvalidAccountData
})?;
load_program_metrics.submit_datapoint(&mut invoke_context.timings);
if let Some(mut source_program) = source_program {
let rent = invoke_context.get_sysvar_cache().get_rent()?;
@ -478,6 +468,20 @@ pub fn process_instruction_deploy(
let state = get_state_mut(program.get_data_mut()?)?;
state.slot = current_slot;
state.is_deployed = true;
if let Some(old_entry) = invoke_context.find_program_in_cache(program.get_key()) {
executor.tx_usage_counter.store(
old_entry.tx_usage_counter.load(Ordering::Relaxed),
Ordering::Relaxed,
);
executor.ix_usage_counter.store(
old_entry.ix_usage_counter.load(Ordering::Relaxed),
Ordering::Relaxed,
);
}
invoke_context
.programs_modified_by_tx
.replenish(*program.get_key(), Arc::new(executor));
Ok(())
}
@ -602,14 +606,13 @@ pub fn process_instruction_inner(
return Err(Box::new(InstructionError::InvalidArgument));
}
let mut get_or_create_executor_time = Measure::start("get_or_create_executor_time");
let (loaded_program, load_program_metrics) = load_program_from_account(
&invoke_context.feature_set,
invoke_context.get_compute_budget(),
invoke_context.get_log_collector(),
&program,
false, /* debugging_features */
)?;
load_program_metrics.submit_datapoint(&mut invoke_context.timings);
let loaded_program = invoke_context
.find_program_in_cache(program.get_key())
.ok_or(InstructionError::InvalidAccountData)?;
if loaded_program.is_tombstone() {
return Err(Box::new(InstructionError::InvalidAccountData));
}
get_or_create_executor_time.stop();
saturating_add_assign!(
invoke_context.timings.get_or_create_executor_us,
@ -651,6 +654,50 @@ mod tests {
std::{fs::File, io::Read, path::Path},
};
pub fn load_all_invoked_programs(invoke_context: &mut InvokeContext) {
let mut load_program_metrics = LoadProgramMetrics::default();
let num_accounts = invoke_context.transaction_context.get_number_of_accounts();
for index in 0..num_accounts {
let account = invoke_context
.transaction_context
.get_account_at_index(index)
.expect("Failed to get the account")
.borrow();
let owner = account.owner();
if loader_v4::check_id(owner) {
let pubkey = invoke_context
.transaction_context
.get_key_of_account_at_index(index)
.expect("Failed to get account key");
if let Some(programdata) =
account.data().get(LoaderV4State::program_data_offset()..)
{
if let Ok(loaded_program) = LoadedProgram::new(
&loader_v4::id(),
invoke_context
.programs_modified_by_tx
.environments
.program_runtime_v2
.clone(),
0,
0,
None,
programdata,
account.data().len(),
&mut load_program_metrics,
) {
invoke_context.programs_modified_by_tx.set_slot_for_tests(0);
invoke_context
.programs_modified_by_tx
.replenish(*pubkey, Arc::new(loaded_program));
}
}
}
}
}
fn process_instruction(
program_indices: Vec<IndexOfAccount>,
instruction_data: &[u8],
@ -676,7 +723,16 @@ mod tests {
instruction_accounts,
expected_result,
super::process_instruction,
|_invoke_context| {},
|invoke_context| {
invoke_context
.programs_modified_by_tx
.environments
.program_runtime_v2 = Arc::new(create_program_runtime_environment_v2(
&ComputeBudget::default(),
false,
));
load_all_invoked_programs(invoke_context);
},
|_invoke_context| {},
)
}
@ -1447,7 +1503,7 @@ mod tests {
load_program_account_from_elf(false, None, "rodata"),
),
(
program_address,
Pubkey::new_unique(),
load_program_account_from_elf(true, None, "invalid"),
),
];

71
runtime/src/bank.rs
View File

@ -109,7 +109,7 @@ use {
invoke_context::ProcessInstructionWithContext,
loaded_programs::{
LoadProgramMetrics, LoadedProgram, LoadedProgramMatchCriteria, LoadedProgramType,
LoadedPrograms, LoadedProgramsForTxBatch, WorkingSlot,
LoadedPrograms, LoadedProgramsForTxBatch, WorkingSlot, DELAY_VISIBILITY_SLOT_OFFSET,
},
log_collector::LogCollector,
message_processor::MessageProcessor,
@ -150,7 +150,7 @@ use {
inflation::Inflation,
instruction::InstructionError,
lamports::LamportsError,
loader_v4,
loader_v4::{self, LoaderV4State},
message::{AccountKeys, SanitizedMessage},
native_loader,
native_token::LAMPORTS_PER_SOL,
@ -305,8 +305,10 @@ impl BankRc {
enum ProgramAccountLoadResult {
AccountNotFound,
InvalidAccountData,
InvalidV4Program,
ProgramOfLoaderV1orV2(AccountSharedData),
ProgramOfLoaderV3(AccountSharedData, AccountSharedData, Slot),
ProgramOfLoaderV4(AccountSharedData, Slot),
}
pub struct LoadAndExecuteTransactionsOutput {
@ -4602,6 +4604,14 @@ impl Bank {
program_account.owner()
));
if loader_v4::check_id(program_account.owner()) {
return solana_loader_v4_program::get_state(program_account.data())
.ok()
.and_then(|state| state.is_deployed.then_some(state.slot))
.map(|slot| ProgramAccountLoadResult::ProgramOfLoaderV4(program_account, slot))
.unwrap_or(ProgramAccountLoadResult::InvalidV4Program);
}
if !bpf_loader_upgradeable::check_id(program_account.owner()) {
return ProgramAccountLoadResult::ProgramOfLoaderV1orV2(program_account);
}
@ -4631,11 +4641,11 @@ impl Bank {
}
pub fn load_program(&self, pubkey: &Pubkey) -> Arc<LoadedProgram> {
let program_runtime_environment_v1 = self
let environments = self
.loaded_programs_cache
.read()
.unwrap()
.program_runtime_environment_v1
.environments
.clone();
let mut load_program_metrics = LoadProgramMetrics {
@ -4662,7 +4672,7 @@ impl Bank {
program_account.owner(),
program_account.data().len(),
0,
program_runtime_environment_v1.clone(),
environments.program_runtime_v1.clone(),
)
}
@ -4686,14 +4696,43 @@ impl Bank {
.len()
.saturating_add(programdata_account.data().len()),
slot,
program_runtime_environment_v1.clone(),
environments.program_runtime_v1.clone(),
)
}),
ProgramAccountLoadResult::ProgramOfLoaderV4(program_account, slot) => {
let loaded_program = program_account
.data()
.get(LoaderV4State::program_data_offset()..)
.and_then(|elf_bytes| {
LoadedProgram::new(
&loader_v4::id(),
environments.program_runtime_v2.clone(),
slot,
slot.saturating_add(DELAY_VISIBILITY_SLOT_OFFSET),
None,
elf_bytes,
program_account.data().len(),
&mut load_program_metrics,
)
.ok()
})
.unwrap_or(LoadedProgram::new_tombstone(
self.slot,
LoadedProgramType::FailedVerification(environments.program_runtime_v2),
));
Ok(loaded_program)
}
ProgramAccountLoadResult::InvalidV4Program => Ok(LoadedProgram::new_tombstone(
self.slot,
LoadedProgramType::FailedVerification(environments.program_runtime_v2),
)),
}
.unwrap_or_else(|_| {
LoadedProgram::new_tombstone(
self.slot,
LoadedProgramType::FailedVerification(program_runtime_environment_v1),
LoadedProgramType::FailedVerification(environments.program_runtime_v1),
)
});
@ -4774,8 +4813,14 @@ impl Bank {
let (blockhash, lamports_per_signature) = self.last_blockhash_and_lamports_per_signature();
let mut executed_units = 0u64;
let mut programs_modified_by_tx = LoadedProgramsForTxBatch::new(self.slot);
let mut programs_updated_only_for_global_cache = LoadedProgramsForTxBatch::new(self.slot);
let mut programs_modified_by_tx = LoadedProgramsForTxBatch::new(
self.slot,
programs_loaded_for_tx_batch.environments.clone(),
);
let mut programs_updated_only_for_global_cache = LoadedProgramsForTxBatch::new(
self.slot,
programs_loaded_for_tx_batch.environments.clone(),
);
let mut process_message_time = Measure::start("process_message_time");
let process_result = MessageProcessor::process_message(
tx.message(),
@ -8012,10 +8057,10 @@ impl Bank {
)
.unwrap();
let mut loaded_programs_cache = self.loaded_programs_cache.write().unwrap();
if *loaded_programs_cache.program_runtime_environment_v1
if *loaded_programs_cache.environments.program_runtime_v1
!= program_runtime_environment_v1
{
loaded_programs_cache.program_runtime_environment_v1 =
loaded_programs_cache.environments.program_runtime_v1 =
Arc::new(program_runtime_environment_v1);
}
let program_runtime_environment_v2 =
@ -8023,10 +8068,10 @@ impl Bank {
&self.runtime_config.compute_budget.unwrap_or_default(),
false, /* debugging_features */
);
if *loaded_programs_cache.program_runtime_environment_v2
if *loaded_programs_cache.environments.program_runtime_v2
!= program_runtime_environment_v2
{
loaded_programs_cache.program_runtime_environment_v2 =
loaded_programs_cache.environments.program_runtime_v2 =
Arc::new(program_runtime_environment_v2);
}
loaded_programs_cache.prune_feature_set_transition();