diff --git a/Cargo.lock b/Cargo.lock index 07a2dbd43..cee039681 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7144,9 +7144,9 @@ dependencies = [ [[package]] name = "solana_rbpf" -version = "0.2.39" +version = "0.2.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199a2e8ced67a32b47695256509d5a62b39cb7340f52f0a5d8de99f4dc2aeb43" +checksum = "1a5735b8c9defc3723162321a61ef738d34168401eeef213f62a32809739b0f5" dependencies = [ "byteorder", "combine", diff --git a/Cargo.toml b/Cargo.toml index 8a1a476ce..5fac23fd0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -287,7 +287,7 @@ signal-hook = "0.3.14" smpl_jwt = "0.7.1" socket2 = "0.4.7" soketto = "0.7" -solana_rbpf = "=0.2.39" +solana_rbpf = "=0.2.40" solana-account-decoder = { path = "account-decoder", version = "=1.16.0" } solana-address-lookup-table-program = { path = "programs/address-lookup-table", version = "=1.16.0" } solana-banks-client = { path = "banks-client", version = "=1.16.0" } diff --git a/programs/bpf_loader/src/allocator_bump.rs b/programs/bpf_loader/src/allocator_bump.rs index cd954d4b6..f5bca8ca5 100644 --- a/programs/bpf_loader/src/allocator_bump.rs +++ b/programs/bpf_loader/src/allocator_bump.rs @@ -26,8 +26,8 @@ impl BpfAllocator { } } - pub fn get_heap(&mut self) -> &mut [u8] { - self.heap.as_slice_mut() + pub fn heap_mut(&mut self) -> &mut AlignedMemory { + &mut self.heap } } diff --git a/programs/bpf_loader/src/lib.rs b/programs/bpf_loader/src/lib.rs index bea7a4c6f..b3227429c 100644 --- a/programs/bpf_loader/src/lib.rs +++ b/programs/bpf_loader/src/lib.rs @@ -28,9 +28,10 @@ use { }, solana_rbpf::{ aligned_memory::AlignedMemory, - ebpf::{HOST_ALIGN, MM_HEAP_START}, + ebpf::{self, HOST_ALIGN, MM_HEAP_START}, + elf::Executable, error::{EbpfError, UserDefinedError}, - memory_region::MemoryRegion, + memory_region::{MemoryCowCallback, MemoryMapping, MemoryRegion}, verifier::{RequisiteVerifier, VerifierError}, vm::{ContextObject, EbpfVm, ProgramResult, VerifiedExecutable}, }, @@ -38,7 +39,7 @@ use { bpf_loader, bpf_loader_deprecated, bpf_loader_upgradeable::{self, UpgradeableLoaderState}, clock::Slot, - entrypoint::{HEAP_LENGTH, SUCCESS}, + entrypoint::SUCCESS, feature_set::{ cap_accounts_data_allocations_per_transaction, cap_bpf_program_instruction_accounts, check_slice_translation_size, delay_visibility_of_program_deployment, @@ -64,6 +65,7 @@ use { std::{ cell::{RefCell, RefMut}, fmt::Debug, + mem, rc::Rc, sync::Arc, }, @@ -296,21 +298,19 @@ fn check_loader_id(id: &Pubkey) -> bool { } /// Create the SBF virtual machine -pub fn create_vm<'a, 'b>( +pub fn create_ebpf_vm<'a, 'b>( program: &'a VerifiedExecutable>, + stack: &'a mut AlignedMemory, + heap: AlignedMemory, regions: Vec, orig_account_lengths: Vec, invoke_context: &'a mut InvokeContext<'b>, ) -> Result>, EbpfError> { - let compute_budget = invoke_context.get_compute_budget(); - let heap_size = compute_budget.heap_size.unwrap_or(HEAP_LENGTH); let _ = invoke_context.consume_checked( - ((heap_size as u64).saturating_div(32_u64.saturating_mul(1024))) + ((heap.len() as u64).saturating_div(32_u64.saturating_mul(1024))) .saturating_sub(1) - .saturating_mul(compute_budget.heap_cost), + .saturating_mul(invoke_context.get_compute_budget().heap_cost), ); - let heap = - AlignedMemory::::zero_filled(compute_budget.heap_size.unwrap_or(HEAP_LENGTH)); let check_aligned = bpf_loader_deprecated::id() != invoke_context .transaction_context @@ -333,13 +333,76 @@ pub fn create_vm<'a, 'b>( allocator.clone(), ) .map_err(SyscallError::InstructionError)?; - let result = EbpfVm::new( - program, - invoke_context, - allocator.borrow_mut().get_heap(), + let stack_len = stack.len(); + let memory_mapping = create_memory_mapping( + program.get_executable(), + stack, + allocator.borrow_mut().heap_mut(), regions, - ); - result + None, + )?; + + EbpfVm::new(program, invoke_context, memory_mapping, stack_len) +} + +#[macro_export] +macro_rules! create_vm { + ($vm_name:ident, $executable:expr, $stack:ident, $heap:ident, $additional_regions:expr, $orig_account_lengths:expr, $invoke_context:expr) => { + let mut $stack = solana_rbpf::aligned_memory::AlignedMemory::< + { solana_rbpf::ebpf::HOST_ALIGN }, + >::zero_filled($executable.get_executable().get_config().stack_size()); + + // this is needed if the caller passes "&mut invoke_context" to the + // macro. The lint complains that (&mut invoke_context).get_compute_budget() + // does an unnecessary mutable borrow + #[allow(clippy::unnecessary_mut_passed)] + let heap_size = $invoke_context + .get_compute_budget() + .heap_size + .unwrap_or(solana_sdk::entrypoint::HEAP_LENGTH); + let $heap = solana_rbpf::aligned_memory::AlignedMemory::<{ solana_rbpf::ebpf::HOST_ALIGN }>::zero_filled(heap_size); + + let $vm_name = create_ebpf_vm( + $executable, + &mut $stack, + $heap, + $additional_regions, + $orig_account_lengths, + $invoke_context, + ); + }; +} + +fn create_memory_mapping<'a, 'b, C: ContextObject>( + executable: &'a Executable, + stack: &'b mut AlignedMemory<{ HOST_ALIGN }>, + heap: &'b mut AlignedMemory<{ HOST_ALIGN }>, + additional_regions: Vec, + cow_cb: Option, +) -> Result, EbpfError> { + let config = executable.get_config(); + let regions: Vec = vec![ + executable.get_ro_region(), + MemoryRegion::new_writable_gapped( + stack.as_slice_mut(), + ebpf::MM_STACK_START, + if !config.dynamic_stack_frames && config.enable_stack_frame_gaps { + config.stack_frame_size as u64 + } else { + 0 + }, + ), + MemoryRegion::new_writable(heap.as_slice_mut(), ebpf::MM_HEAP_START), + ] + .into_iter() + .chain(additional_regions.into_iter()) + .collect(); + + Ok(if let Some(cow_cb) = cow_cb { + MemoryMapping::new_with_cow(regions, cow_cb, config)? + } else { + MemoryMapping::new(regions, config)? + }) } pub fn process_instruction(invoke_context: &mut InvokeContext) -> Result<(), InstructionError> { @@ -1345,9 +1408,9 @@ fn process_loader_instruction( Ok(()) } -fn execute( - executable: &VerifiedExecutable>, - invoke_context: &mut InvokeContext, +fn execute<'a, 'b: 'a>( + executable: &'a VerifiedExecutable>, + invoke_context: &'a mut InvokeContext<'b>, ) -> Result<(), InstructionError> { let log_collector = invoke_context.get_log_collector(); let stack_height = invoke_context.get_stack_height(); @@ -1373,14 +1436,22 @@ fn execute( let mut execute_time; let execution_result = { let compute_meter_prev = invoke_context.get_remaining(); - let mut vm = match create_vm( + create_vm!( + vm, // We dropped the lifetime tracking in the Executor by setting it to 'static, // thus we need to reintroduce the correct lifetime of InvokeContext here again. - unsafe { std::mem::transmute(executable) }, + unsafe { + mem::transmute::<_, &'a VerifiedExecutable>>( + executable, + ) + }, + stack, + heap, regions, account_lengths, - invoke_context, - ) { + invoke_context + ); + let mut vm = match vm { Ok(info) => info, Err(e) => { ic_logger_msg!(log_collector, "Failed to create SBF VM: {}", e); @@ -1592,11 +1663,28 @@ mod tests { .unwrap(); let input_region = MemoryRegion::new_writable(&mut input_mem, MM_INPUT_START); let mut context_object = TestContextObject { remaining: 10 }; + let mut stack = AlignedMemory::zero_filled( + verified_executable + .get_executable() + .get_config() + .stack_size(), + ); + let mut heap = AlignedMemory::with_capacity(0); + let stack_len = stack.len(); + + let memory_mapping = create_memory_mapping( + verified_executable.get_executable(), + &mut stack, + &mut heap, + vec![input_region], + None, + ) + .unwrap(); let mut vm = EbpfVm::new( &verified_executable, &mut context_object, - &mut [], - vec![input_region], + memory_mapping, + stack_len, ) .unwrap(); vm.execute_program(true).1.unwrap(); diff --git a/programs/bpf_loader/src/syscalls/cpi.rs b/programs/bpf_loader/src/syscalls/cpi.rs index 4e99f30a0..45654be27 100644 --- a/programs/bpf_loader/src/syscalls/cpi.rs +++ b/programs/bpf_loader/src/syscalls/cpi.rs @@ -1333,7 +1333,7 @@ mod tests { ..Config::default() }; let memory_mapping = - MemoryMapping::new(vec![mock_caller_account.region.clone()], &config).unwrap(); + MemoryMapping::new(mock_caller_account.regions.split_off(0), &config).unwrap(); let mut caller_account = mock_caller_account.caller_account(); @@ -1390,7 +1390,7 @@ mod tests { ..Config::default() }; let memory_mapping = - MemoryMapping::new(vec![mock_caller_account.region.clone()], &config).unwrap(); + MemoryMapping::new(mock_caller_account.regions.split_off(0), &config).unwrap(); let data_slice = mock_caller_account.data_slice(); let len_ptr = unsafe { @@ -1608,7 +1608,7 @@ mod tests { vm_addr: u64, data: Vec, len: u64, - region: MemoryRegion, + regions: Vec, } impl MockCallerAccount { @@ -1620,7 +1620,7 @@ mod tests { d[mem::size_of::()..][..data.len()].copy_from_slice(data); // the memory region must include the realloc data - let region = MemoryRegion::new_writable(d.as_mut_slice(), vm_addr); + let regions = vec![MemoryRegion::new_writable(d.as_mut_slice(), vm_addr)]; // caller_account.data must have the actual data length d.truncate(mem::size_of::() + data.len()); @@ -1631,7 +1631,7 @@ mod tests { vm_addr, data: d, len: data.len() as u64, - region, + regions, } } diff --git a/programs/bpf_loader/src/syscalls/mod.rs b/programs/bpf_loader/src/syscalls/mod.rs index e14f8a209..3d7d321e4 100644 --- a/programs/bpf_loader/src/syscalls/mod.rs +++ b/programs/bpf_loader/src/syscalls/mod.rs @@ -1803,12 +1803,15 @@ declare_syscall!( ); #[cfg(test)] +#[allow(clippy::integer_arithmetic)] +#[allow(clippy::indexing_slicing)] mod tests { #[allow(deprecated)] use solana_sdk::sysvar::fees::Fees; use { super::*, crate::BpfAllocator, + core::slice, solana_program_runtime::{invoke_context::InvokeContext, sysvar_cache::SysvarCache}, solana_rbpf::{ aligned_memory::AlignedMemory, @@ -1827,7 +1830,7 @@ mod tests { sysvar::{clock::Clock, epoch_schedule::EpochSchedule, rent::Rent}, transaction_context::TransactionContext, }, - std::{borrow::Cow, cell::RefCell, rc::Rc, str::FromStr}, + std::{borrow::Cow, cell::RefCell, mem, rc::Rc, str::FromStr}, }; macro_rules! assert_access_violation { @@ -1877,6 +1880,7 @@ mod tests { fn test_translate() { const START: u64 = 0x100000000; const LENGTH: u64 = 1000; + let data = vec![0u8; LENGTH as usize]; let addr = data.as_ptr() as u64; let config = Config::default(); @@ -1913,19 +1917,12 @@ mod tests { #[test] fn test_translate_type() { + let config = Config::default(); + // Pubkey let pubkey = solana_sdk::pubkey::new_rand(); - let addr = &pubkey as *const _ as u64; - let config = Config::default(); let memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: addr, - vm_addr: 0x100000000, - vm_addr_end: 0x100000000_u64.saturating_add(std::mem::size_of::() as u64), - len: std::mem::size_of::() as u64, - vm_gap_shift: 63, - is_writable: false, - }], + vec![MemoryRegion::new_readonly(bytes_of(&pubkey), 0x100000000)], &config, ) .unwrap(); @@ -1940,48 +1937,27 @@ mod tests { vec![AccountMeta::new(solana_sdk::pubkey::new_rand(), false)], ); let instruction = StableInstruction::from(instruction); - let addr = &instruction as *const _ as u64; - let mut memory_region = MemoryRegion { - host_addr: addr, - vm_addr: 0x100000000, - vm_addr_end: 0x100000000_u64.saturating_add(std::mem::size_of::() as u64), - len: std::mem::size_of::() as u64, - vm_gap_shift: 63, - is_writable: false, - }; - let mut memory_mapping = MemoryMapping::new(vec![memory_region.clone()], &config).unwrap(); + let memory_region = MemoryRegion::new_readonly(bytes_of(&instruction), 0x100000000); + let memory_mapping = MemoryMapping::new(vec![memory_region], &config).unwrap(); let translated_instruction = translate_type::(&memory_mapping, 0x100000000, true).unwrap(); assert_eq!(instruction, *translated_instruction); - memory_region.len = 1; - let memory_region_index = memory_mapping - .get_regions() - .iter() - .position(|memory_region| memory_region.vm_addr == 0x100000000) - .unwrap(); - memory_mapping - .replace_region(memory_region_index, memory_region) - .unwrap(); - assert!(translate_type::(&memory_mapping, 0x100000000, true).is_err()); + + let memory_region = MemoryRegion::new_readonly(&bytes_of(&instruction)[..1], 0x100000000); + let memory_mapping = MemoryMapping::new(vec![memory_region], &config).unwrap(); + assert!(translate_type::(&memory_mapping, 0x100000000, true).is_err()); } #[test] fn test_translate_slice() { + let config = Config::default(); + // zero len let good_data = vec![1u8, 2, 3, 4, 5]; let data: Vec = vec![]; assert_eq!(0x1 as *const u8, data.as_ptr()); - let addr = good_data.as_ptr() as *const _ as u64; - let config = Config::default(); let memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: addr, - vm_addr: 0x100000000, - vm_addr_end: 0x100000000_u64.saturating_add(good_data.len() as u64), - len: good_data.len() as u64, - vm_gap_shift: 63, - is_writable: false, - }], + vec![MemoryRegion::new_readonly(&good_data, 0x100000000)], &config, ) .unwrap(); @@ -1992,16 +1968,8 @@ mod tests { // u8 let mut data = vec![1u8, 2, 3, 4, 5]; - let addr = data.as_ptr() as *const _ as u64; let memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: addr, - vm_addr: 0x100000000, - vm_addr_end: 0x100000000_u64.saturating_add(data.len() as u64), - len: data.len() as u64, - vm_gap_shift: 63, - is_writable: false, - }], + vec![MemoryRegion::new_readonly(&data, 0x100000000)], &config, ) .unwrap(); @@ -2027,16 +1995,11 @@ mod tests { // u64 let mut data = vec![1u64, 2, 3, 4, 5]; - let addr = data.as_ptr() as *const _ as u64; let memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: addr, - vm_addr: 0x100000000, - vm_addr_end: 0x100000000_u64.saturating_add((data.len() * size_of::()) as u64), - len: (data.len() * size_of::()) as u64, - vm_gap_shift: 63, - is_writable: false, - }], + vec![MemoryRegion::new_readonly( + bytes_of_slice(&data), + 0x100000000, + )], &config, ) .unwrap(); @@ -2052,17 +2015,13 @@ mod tests { // Pubkeys let mut data = vec![solana_sdk::pubkey::new_rand(); 5]; - let addr = data.as_ptr() as *const _ as u64; let memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: addr, - vm_addr: 0x100000000, - vm_addr_end: 0x100000000_u64 - .saturating_add((data.len() * std::mem::size_of::()) as u64), - len: (data.len() * std::mem::size_of::()) as u64, - vm_gap_shift: 63, - is_writable: false, - }], + vec![MemoryRegion::new_readonly( + unsafe { + slice::from_raw_parts(data.as_ptr() as *const u8, mem::size_of::() * 5) + }, + 0x100000000, + )], &config, ) .unwrap(); @@ -2077,17 +2036,9 @@ mod tests { #[test] fn test_translate_string_and_do() { let string = "Gaggablaghblagh!"; - let addr = string.as_ptr() as *const _ as u64; let config = Config::default(); let memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: addr, - vm_addr: 0x100000000, - len: string.len() as u64, - vm_addr_end: 0x100000000_u64.saturating_add(string.len() as u64), - vm_gap_shift: 63, - is_writable: false, - }], + vec![MemoryRegion::new_readonly(string.as_bytes(), 0x100000000)], &config, ) .unwrap(); @@ -2144,17 +2095,9 @@ mod tests { ); let string = "Gaggablaghblagh!"; - let addr = string.as_ptr() as *const _ as u64; let config = Config::default(); let mut memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: addr, - vm_addr: 0x100000000, - len: string.len() as u64, - vm_addr_end: 0x100000000_u64.saturating_add(string.len() as u64), - vm_gap_shift: 63, - is_writable: false, - }], + vec![MemoryRegion::new_readonly(string.as_bytes(), 0x100000000)], &config, ) .unwrap(); @@ -2203,17 +2146,9 @@ mod tests { ); let string = "Gaggablaghblagh!"; - let addr = string.as_ptr() as *const _ as u64; let config = Config::default(); let mut memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: addr, - vm_addr: 0x100000000, - len: string.len() as u64, - vm_addr_end: 0x100000000_u64.saturating_add(string.len() as u64), - vm_gap_shift: 63, - is_writable: false, - }], + vec![MemoryRegion::new_readonly(string.as_bytes(), 0x100000000)], &config, ) .unwrap(); @@ -2331,17 +2266,9 @@ mod tests { let cost = invoke_context.get_compute_budget().log_pubkey_units; let pubkey = Pubkey::from_str("MoqiU1vryuCGQSxFKA1SZ316JdLEFFhoAu6cKUNk7dN").unwrap(); - let addr = pubkey.as_ref().first().unwrap() as *const _ as u64; let config = Config::default(); let mut memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: addr, - vm_addr: 0x100000000, - len: 32, - vm_addr_end: 0x100000000_u64.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }], + vec![MemoryRegion::new_readonly(bytes_of(&pubkey), 0x100000000)], &config, ) .unwrap(); @@ -2657,44 +2584,16 @@ mod tests { len: bytes2.len(), }; let bytes_to_hash = [mock_slice1, mock_slice2]; - let hash_result = [0; HASH_BYTES]; + let mut hash_result = [0; HASH_BYTES]; let ro_len = bytes_to_hash.len() as u64; let ro_va = 0x100000000; let rw_va = 0x200000000; let mut memory_mapping = MemoryMapping::new( vec![ - MemoryRegion { - host_addr: bytes_to_hash.as_ptr() as *const _ as u64, - vm_addr: ro_va, - len: 32, - vm_addr_end: ro_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: hash_result.as_ptr() as *const _ as u64, - vm_addr: rw_va, - len: HASH_BYTES as u64, - vm_addr_end: rw_va.saturating_add(HASH_BYTES as u64), - vm_gap_shift: 63, - is_writable: true, - }, - MemoryRegion { - host_addr: bytes1.as_ptr() as *const _ as u64, - vm_addr: bytes_to_hash[0].vm_addr, - len: bytes1.len() as u64, - vm_addr_end: bytes_to_hash[0].vm_addr.saturating_add(bytes1.len() as u64), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: bytes2.as_ptr() as *const _ as u64, - vm_addr: bytes_to_hash[1].vm_addr, - len: bytes2.len() as u64, - vm_addr_end: bytes_to_hash[1].vm_addr.saturating_add(bytes2.len() as u64), - vm_gap_shift: 63, - is_writable: false, - }, + MemoryRegion::new_readonly(bytes_of_slice(&bytes_to_hash), ro_va), + MemoryRegion::new_writable(bytes_of_slice_mut(&mut hash_result), rw_va), + MemoryRegion::new_readonly(bytes1.as_bytes(), bytes_to_hash[0].vm_addr), + MemoryRegion::new_readonly(bytes2.as_bytes(), bytes_to_hash[1].vm_addr), ], &config, ) @@ -2807,22 +2706,8 @@ mod tests { let mut memory_mapping = MemoryMapping::new( vec![ - MemoryRegion { - host_addr: valid_bytes.as_ptr() as *const _ as u64, - vm_addr: valid_bytes_va, - len: 32, - vm_addr_end: valid_bytes_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: invalid_bytes.as_ptr() as *const _ as u64, - vm_addr: invalid_bytes_va, - len: 32, - vm_addr_end: invalid_bytes_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, + MemoryRegion::new_readonly(&valid_bytes, valid_bytes_va), + MemoryRegion::new_readonly(&invalid_bytes, invalid_bytes_va), ], &config, ) @@ -2906,22 +2791,8 @@ mod tests { let mut memory_mapping = MemoryMapping::new( vec![ - MemoryRegion { - host_addr: valid_bytes.as_ptr() as *const _ as u64, - vm_addr: valid_bytes_va, - len: 32, - vm_addr_end: valid_bytes_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: invalid_bytes.as_ptr() as *const _ as u64, - vm_addr: invalid_bytes_va, - len: 32, - vm_addr_end: invalid_bytes_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, + MemoryRegion::new_readonly(&valid_bytes, valid_bytes_va), + MemoryRegion::new_readonly(&invalid_bytes, invalid_bytes_va), ], &config, ) @@ -3013,51 +2884,16 @@ mod tests { 60, 87, 144, 161, 146, 42, 34, 91, 155, 158, 189, 121, 79, ]; let invalid_point_va = 0x400000000; - let result_point: [u8; 32] = [0; 32]; + let mut result_point: [u8; 32] = [0; 32]; let result_point_va = 0x500000000; let mut memory_mapping = MemoryMapping::new( vec![ - MemoryRegion { - host_addr: left_point.as_ptr() as *const _ as u64, - vm_addr: left_point_va, - len: 32, - vm_addr_end: left_point_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: right_point.as_ptr() as *const _ as u64, - vm_addr: right_point_va, - len: 32, - vm_addr_end: right_point_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: scalar.as_ptr() as *const _ as u64, - vm_addr: scalar_va, - len: 32, - vm_addr_end: scalar_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: invalid_point.as_ptr() as *const _ as u64, - vm_addr: invalid_point_va, - len: 32, - vm_addr_end: invalid_point_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: result_point.as_ptr() as *const _ as u64, - vm_addr: result_point_va, - len: 32, - vm_addr_end: result_point_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: true, - }, + MemoryRegion::new_readonly(bytes_of_slice(&left_point), left_point_va), + MemoryRegion::new_readonly(bytes_of_slice(&right_point), right_point_va), + MemoryRegion::new_readonly(bytes_of_slice(&scalar), scalar_va), + MemoryRegion::new_readonly(bytes_of_slice(&invalid_point), invalid_point_va), + MemoryRegion::new_writable(bytes_of_slice_mut(&mut result_point), result_point_va), ], &config, ) @@ -3225,51 +3061,16 @@ mod tests { 60, 87, 144, 161, 146, 42, 34, 91, 155, 158, 189, 121, 79, ]; let invalid_point_va = 0x400000000; - let result_point: [u8; 32] = [0; 32]; + let mut result_point: [u8; 32] = [0; 32]; let result_point_va = 0x500000000; let mut memory_mapping = MemoryMapping::new( vec![ - MemoryRegion { - host_addr: left_point.as_ptr() as *const _ as u64, - vm_addr: left_point_va, - len: 32, - vm_addr_end: left_point_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: right_point.as_ptr() as *const _ as u64, - vm_addr: right_point_va, - len: 32, - vm_addr_end: right_point_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: scalar.as_ptr() as *const _ as u64, - vm_addr: scalar_va, - len: 32, - vm_addr_end: scalar_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: invalid_point.as_ptr() as *const _ as u64, - vm_addr: invalid_point_va, - len: 32, - vm_addr_end: invalid_point_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: result_point.as_ptr() as *const _ as u64, - vm_addr: result_point_va, - len: 32, - vm_addr_end: result_point_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: true, - }, + MemoryRegion::new_readonly(bytes_of_slice(&left_point), left_point_va), + MemoryRegion::new_readonly(bytes_of_slice(&right_point), right_point_va), + MemoryRegion::new_readonly(bytes_of_slice(&scalar), scalar_va), + MemoryRegion::new_readonly(bytes_of_slice(&invalid_point), invalid_point_va), + MemoryRegion::new_writable(bytes_of_slice_mut(&mut result_point), result_point_va), ], &config, ) @@ -3453,43 +3254,15 @@ mod tests { let ristretto_points = [ristretto_point_x, ristretto_point_y]; let ristretto_points_va = 0x300000000; - let result_point: [u8; 32] = [0; 32]; + let mut result_point: [u8; 32] = [0; 32]; let result_point_va = 0x400000000; let mut memory_mapping = MemoryMapping::new( vec![ - MemoryRegion { - host_addr: scalars.as_ptr() as *const _ as u64, - vm_addr: scalars_va, - len: 64, - vm_addr_end: scalars_va.saturating_add(64), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: edwards_points.as_ptr() as *const _ as u64, - vm_addr: edwards_points_va, - len: 64, - vm_addr_end: edwards_points_va.saturating_add(64), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: ristretto_points.as_ptr() as *const _ as u64, - vm_addr: ristretto_points_va, - len: 64, - vm_addr_end: ristretto_points_va.saturating_add(64), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: result_point.as_ptr() as *const _ as u64, - vm_addr: result_point_va, - len: 32, - vm_addr_end: result_point_va.saturating_add(32), - vm_gap_shift: 63, - is_writable: true, - }, + MemoryRegion::new_readonly(bytes_of_slice(&scalars), scalars_va), + MemoryRegion::new_readonly(bytes_of_slice(&edwards_points), edwards_points_va), + MemoryRegion::new_readonly(bytes_of_slice(&ristretto_points), ristretto_points_va), + MemoryRegion::new_writable(bytes_of_slice_mut(&mut result_point), result_point_va), ], &config, ) @@ -3615,18 +3388,14 @@ mod tests { // Test clock sysvar { - let got_clock = Clock::default(); + let mut got_clock = Clock::default(); let got_clock_va = 0x100000000; let mut memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: &got_clock as *const _ as u64, - vm_addr: got_clock_va, - len: size_of::() as u64, - vm_addr_end: got_clock_va.saturating_add(size_of::() as u64), - vm_gap_shift: 63, - is_writable: true, - }], + vec![MemoryRegion::new_writable( + bytes_of_mut(&mut got_clock), + got_clock_va, + )], &config, ) .unwrap(); @@ -3656,19 +3425,14 @@ mod tests { // Test epoch_schedule sysvar { - let got_epochschedule = EpochSchedule::default(); + let mut got_epochschedule = EpochSchedule::default(); let got_epochschedule_va = 0x100000000; let mut memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: &got_epochschedule as *const _ as u64, - vm_addr: got_epochschedule_va, - len: size_of::() as u64, - vm_addr_end: got_epochschedule_va - .saturating_add(size_of::() as u64), - vm_gap_shift: 63, - is_writable: true, - }], + vec![MemoryRegion::new_writable( + bytes_of_mut(&mut got_epochschedule), + got_epochschedule_va, + )], &config, ) .unwrap(); @@ -3699,18 +3463,14 @@ mod tests { // Test fees sysvar { - let got_fees = Fees::default(); + let mut got_fees = Fees::default(); let got_fees_va = 0x100000000; let mut memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: &got_fees as *const _ as u64, - vm_addr: got_fees_va, - len: size_of::() as u64, - vm_addr_end: got_fees_va.saturating_add(size_of::() as u64), - vm_gap_shift: 63, - is_writable: true, - }], + vec![MemoryRegion::new_writable( + bytes_of_mut(&mut got_fees), + got_fees_va, + )], &config, ) .unwrap(); @@ -3736,18 +3496,14 @@ mod tests { // Test rent sysvar { - let got_rent = create_filled_type::(true); + let mut got_rent = create_filled_type::(true); let got_rent_va = 0x100000000; let mut memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: &got_rent as *const _ as u64, - vm_addr: got_rent_va, - len: size_of::() as u64, - vm_addr_end: got_rent_va.saturating_add(size_of::() as u64), - vm_gap_shift: 63, - is_writable: true, - }], + vec![MemoryRegion::new_writable( + bytes_of_mut(&mut got_rent), + got_rent_va, + )], &config, ) .unwrap(); @@ -3788,45 +3544,15 @@ mod tests { const SEED_VA: u64 = 0x500000000; let config = Config::default(); - let address = Pubkey::default(); - let bump_seed = 0; - let mut mock_slices = Vec::with_capacity(seeds.len()); + let mut address = Pubkey::default(); + let mut bump_seed = 0; let mut regions = vec![ - MemoryRegion { - host_addr: mock_slices.as_ptr() as u64, - vm_addr: SEEDS_VA, - len: (seeds.len().saturating_mul(size_of::()) as u64), - vm_addr_end: SEEDS_VA - .saturating_add(seeds.len().saturating_mul(size_of::()) as u64), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: program_id.as_ref().as_ptr() as u64, - vm_addr: PROGRAM_ID_VA, - len: 32, - vm_addr_end: PROGRAM_ID_VA.saturating_add(32), - vm_gap_shift: 63, - is_writable: false, - }, - MemoryRegion { - host_addr: address.as_ref().as_ptr() as u64, - vm_addr: ADDRESS_VA, - len: 32, - vm_addr_end: ADDRESS_VA.saturating_add(32), - vm_gap_shift: 63, - is_writable: true, - }, - MemoryRegion { - host_addr: &bump_seed as *const u8 as u64, - vm_addr: BUMP_SEED_VA, - len: 32, - vm_addr_end: BUMP_SEED_VA.saturating_add(32), - vm_gap_shift: 63, - is_writable: true, - }, + MemoryRegion::new_readonly(bytes_of(program_id), PROGRAM_ID_VA), + MemoryRegion::new_writable(bytes_of_mut(&mut address), ADDRESS_VA), + MemoryRegion::new_writable(bytes_of_mut(&mut bump_seed), BUMP_SEED_VA), ]; + let mut mock_slices = Vec::with_capacity(seeds.len()); for (i, seed) in seeds.iter().enumerate() { let vm_addr = SEED_VA.saturating_add((i as u64).saturating_mul(0x100000000)); let mock_slice = MockSlice { @@ -3834,15 +3560,12 @@ mod tests { len: seed.len(), }; mock_slices.push(mock_slice); - regions.push(MemoryRegion { - host_addr: seed.as_ptr() as u64, - vm_addr, - len: seed.len() as u64, - vm_addr_end: vm_addr.saturating_add(seed.len() as u64), - vm_gap_shift: 63, - is_writable: false, - }); + regions.push(MemoryRegion::new_readonly(bytes_of_slice(seed), vm_addr)); } + regions.push(MemoryRegion::new_readonly( + bytes_of_slice(&mock_slices), + SEEDS_VA, + )); let mut memory_mapping = MemoryMapping::new(regions, &config).unwrap(); let mut result = ProgramResult::Ok(0); @@ -4012,14 +3735,7 @@ mod tests { let mut memory = [0u8; END_OFFSET]; let config = Config::default(); let mut memory_mapping = MemoryMapping::new( - vec![MemoryRegion { - host_addr: memory.as_mut_ptr() as u64, - vm_addr: VM_BASE_ADDRESS, - len: END_OFFSET as u64, - vm_addr_end: VM_BASE_ADDRESS.saturating_add(END_OFFSET as u64), - vm_gap_shift: 63, - is_writable: true, - }], + vec![MemoryRegion::new_writable(&mut memory, VM_BASE_ADDRESS)], &config, ) .unwrap(); @@ -4334,4 +4050,24 @@ mod tests { fn test_check_type_assumptions() { check_type_assumptions(); } + + fn bytes_of(val: &T) -> &[u8] { + let size = mem::size_of::(); + unsafe { slice::from_raw_parts(std::slice::from_ref(val).as_ptr().cast(), size) } + } + + fn bytes_of_mut(val: &mut T) -> &mut [u8] { + let size = mem::size_of::(); + unsafe { slice::from_raw_parts_mut(slice::from_mut(val).as_mut_ptr().cast(), size) } + } + + pub fn bytes_of_slice(val: &[T]) -> &[u8] { + let size = val.len().wrapping_mul(mem::size_of::()); + unsafe { slice::from_raw_parts(val.as_ptr().cast(), size) } + } + + pub fn bytes_of_slice_mut(val: &mut [T]) -> &mut [u8] { + let size = val.len().wrapping_mul(mem::size_of::()); + unsafe { slice::from_raw_parts_mut(val.as_mut_ptr().cast(), size) } + } } diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 1c4f24a7c..8d33f7c35 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -6325,9 +6325,9 @@ dependencies = [ [[package]] name = "solana_rbpf" -version = "0.2.39" +version = "0.2.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199a2e8ced67a32b47695256509d5a62b39cb7340f52f0a5d8de99f4dc2aeb43" +checksum = "1a5735b8c9defc3723162321a61ef738d34168401eeef213f62a32809739b0f5" dependencies = [ "byteorder 1.4.3", "combine", diff --git a/programs/sbf/Cargo.toml b/programs/sbf/Cargo.toml index 9cd03cb13..1d7d8b21e 100644 --- a/programs/sbf/Cargo.toml +++ b/programs/sbf/Cargo.toml @@ -24,7 +24,7 @@ num-traits = "0.2" rand = "0.7" serde = "1.0.112" serde_json = "1.0.56" -solana_rbpf = "=0.2.39" +solana_rbpf = "=0.2.40" solana-account-decoder = { path = "../../account-decoder", version = "=1.16.0" } solana-address-lookup-table-program = { path = "../../programs/address-lookup-table", version = "=1.16.0" } solana-bpf-loader-program = { path = "../bpf_loader", version = "=1.16.0" } diff --git a/programs/sbf/benches/bpf_loader.rs b/programs/sbf/benches/bpf_loader.rs index 93a7da737..508de137c 100644 --- a/programs/sbf/benches/bpf_loader.rs +++ b/programs/sbf/benches/bpf_loader.rs @@ -1,6 +1,8 @@ #![feature(test)] #![cfg(feature = "sbf_c")] +use {solana_rbpf::memory_region::MemoryState, std::slice}; + extern crate test; #[macro_use] extern crate solana_bpf_loader_program; @@ -8,7 +10,7 @@ extern crate solana_bpf_loader_program; use { byteorder::{ByteOrder, LittleEndian, WriteBytesExt}, solana_bpf_loader_program::{ - create_vm, serialization::serialize_parameters, syscalls::create_loader, + create_ebpf_vm, create_vm, serialization::serialize_parameters, syscalls::create_loader, }, solana_measure::measure::Measure, solana_program_runtime::{ @@ -88,13 +90,16 @@ fn bench_program_alu(bencher: &mut Bencher) { .unwrap(); verified_executable.jit_compile().unwrap(); - let mut vm = create_vm( + create_vm!( + vm, &verified_executable, + stack, + heap, vec![MemoryRegion::new_writable(&mut inner_iter, MM_INPUT_START)], vec![], - invoke_context, - ) - .unwrap(); + invoke_context + ); + let mut vm = vm.unwrap(); println!("Interpreted:"); vm.env @@ -188,17 +193,6 @@ fn bench_create_vm(bencher: &mut Bencher) { const BUDGET: u64 = 200_000; invoke_context.mock_set_remaining(BUDGET); - // Serialize account data - let (_serialized, regions, account_lengths) = serialize_parameters( - invoke_context.transaction_context, - invoke_context - .transaction_context - .get_current_instruction_context() - .unwrap(), - true, // should_cap_ix_accounts - ) - .unwrap(); - let loader = create_loader( &invoke_context.feature_set, &ComputeBudget::default(), @@ -213,14 +207,28 @@ fn bench_create_vm(bencher: &mut Bencher) { VerifiedExecutable::::from_executable(executable) .unwrap(); + // Serialize account data + let (_serialized, regions, account_lengths) = serialize_parameters( + invoke_context.transaction_context, + invoke_context + .transaction_context + .get_current_instruction_context() + .unwrap(), + true, // should_cap_ix_accounts + ) + .unwrap(); + bencher.iter(|| { - let _ = create_vm( + create_vm!( + vm, &verified_executable, - regions.clone(), + stack, + heap, + clone_regions(®ions), account_lengths.clone(), - invoke_context, - ) - .unwrap(); + invoke_context + ); + let _ = vm.unwrap(); }); }); } @@ -258,13 +266,16 @@ fn bench_instruction_count_tuner(_bencher: &mut Bencher) { VerifiedExecutable::::from_executable(executable) .unwrap(); - let mut vm = create_vm( + create_vm!( + vm, &verified_executable, + stack, + heap, regions, account_lengths, - invoke_context, - ) - .unwrap(); + invoke_context + ); + let mut vm = vm.unwrap(); let mut measure = Measure::start("tune"); let (instructions, _result) = vm.execute_program(true); @@ -283,3 +294,29 @@ fn bench_instruction_count_tuner(_bencher: &mut Bencher) { ); }); } + +fn clone_regions(regions: &[MemoryRegion]) -> Vec { + unsafe { + regions + .iter() + .map(|region| match region.state.get() { + MemoryState::Readable => MemoryRegion::new_readonly( + slice::from_raw_parts(region.host_addr.get() as *const _, region.len as usize), + region.vm_addr, + ), + MemoryState::Writable => MemoryRegion::new_writable( + slice::from_raw_parts_mut( + region.host_addr.get() as *mut _, + region.len as usize, + ), + region.vm_addr, + ), + MemoryState::Cow(id) => MemoryRegion::new_cow( + slice::from_raw_parts(region.host_addr.get() as *const _, region.len as usize), + region.vm_addr, + id, + ), + }) + .collect() + } +} diff --git a/programs/sbf/tests/programs.rs b/programs/sbf/tests/programs.rs index 24876a99b..e5ce5a765 100644 --- a/programs/sbf/tests/programs.rs +++ b/programs/sbf/tests/programs.rs @@ -54,7 +54,7 @@ use { }; use { solana_bpf_loader_program::{ - create_vm, + create_ebpf_vm, create_vm, serialization::{deserialize_parameters, serialize_parameters}, syscalls::create_loader, }, @@ -134,13 +134,16 @@ fn run_program(name: &str) -> u64 { .unwrap(); { - let mut vm = create_vm( + create_vm!( + vm, &verified_executable, + stack, + heap, regions, account_lengths.clone(), - invoke_context, - ) - .unwrap(); + invoke_context + ); + let mut vm = vm.unwrap(); let (compute_units_consumed, result) = vm.execute_program(i == 0); assert_eq!(SUCCESS, result.unwrap()); if i == 1 { diff --git a/rbpf-cli/src/main.rs b/rbpf-cli/src/main.rs index 7aecc1ce0..820dc492a 100644 --- a/rbpf-cli/src/main.rs +++ b/rbpf-cli/src/main.rs @@ -3,7 +3,7 @@ use { serde::{Deserialize, Serialize}, serde_json::Result, solana_bpf_loader_program::{ - create_vm, serialization::serialize_parameters, syscalls::create_loader, + create_ebpf_vm, create_vm, serialization::serialize_parameters, syscalls::create_loader, }, solana_program_runtime::{ compute_budget::ComputeBudget, @@ -292,14 +292,16 @@ before execting it in the virtual machine.", } _ => {} } - - let mut vm = create_vm( + create_vm!( + vm, &verified_executable, + stack, + heap, regions, account_lengths, - &mut invoke_context, - ) - .unwrap(); + &mut invoke_context + ); + let mut vm = vm.unwrap(); let start_time = Instant::now(); if matches.value_of("use").unwrap() == "debugger" { vm.debug_port = Some(matches.value_of("port").unwrap().parse::().unwrap());