From f8516b677a360347819441ca3bc51d63b9b6e79a Mon Sep 17 00:00:00 2001 From: jackcmay Date: Fri, 19 Oct 2018 18:28:38 -0700 Subject: [PATCH] Load program data in chunks (#1556) Load program data in chunks --- programs/native/bpf_loader/src/lib.rs | 106 +++------------------ programs/native/lua_loader/src/lib.rs | 47 +++------- src/native_loader.rs | 8 +- tests/programs.rs | 130 ++++++++++++++++++-------- 4 files changed, 122 insertions(+), 169 deletions(-) diff --git a/programs/native/bpf_loader/src/lib.rs b/programs/native/bpf_loader/src/lib.rs index b14cec6614..5a55f17ce8 100644 --- a/programs/native/bpf_loader/src/lib.rs +++ b/programs/native/bpf_loader/src/lib.rs @@ -2,51 +2,23 @@ pub mod bpf_verifier; extern crate bincode; extern crate byteorder; -extern crate elf; extern crate env_logger; -extern crate libc; #[macro_use] extern crate log; extern crate rbpf; -#[macro_use] -extern crate serde_derive; extern crate solana_program_interface; -use bincode::{deserialize, serialize}; +use bincode::deserialize; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; use solana_program_interface::account::KeyedAccount; use solana_program_interface::loader_instruction::LoaderInstruction; use solana_program_interface::pubkey::Pubkey; -use std::env; use std::io::prelude::*; use std::io::Error; use std::mem; -use std::path::PathBuf; use std::str; use std::sync::{Once, ONCE_INIT}; -/// Dynamic link library prefixs -const PLATFORM_FILE_PREFIX_BPF: &str = ""; - -/// Dynamic link library file extension specific to the platform -const PLATFORM_FILE_EXTENSION_BPF: &str = "o"; - -/// Section name -pub const PLATFORM_SECTION_RS: &str = ".text,entrypoint"; -pub const PLATFORM_SECTION_C: &str = ".text.entrypoint"; - -fn create_path(name: &str) -> PathBuf { - let pathbuf = { - let current_exe = env::current_exe().unwrap(); - PathBuf::from(current_exe.parent().unwrap().parent().unwrap()) - }; - - pathbuf.join( - PathBuf::from(PLATFORM_FILE_PREFIX_BPF.to_string() + name) - .with_extension(PLATFORM_FILE_EXTENSION_BPF), - ) -} - fn create_vm(prog: &[u8]) -> Result { let mut vm = rbpf::EbpfVmRaw::new(None)?; vm.set_verifier(bpf_verifier::check)?; @@ -72,7 +44,7 @@ fn dump_prog(name: &str, prog: &[u8]) { } } -fn serialize_state(keyed_accounts: &mut [KeyedAccount], data: &[u8]) -> Vec { +fn serialize_parameters(keyed_accounts: &mut [KeyedAccount], data: &[u8]) -> Vec { assert_eq!(32, mem::size_of::()); let mut v: Vec = Vec::new(); @@ -91,7 +63,7 @@ fn serialize_state(keyed_accounts: &mut [KeyedAccount], data: &[u8]) -> Vec v } -fn deserialize_state(keyed_accounts: &mut [KeyedAccount], buffer: &[u8]) { +fn deserialize_parameters(keyed_accounts: &mut [KeyedAccount], buffer: &[u8]) { assert_eq!(32, mem::size_of::()); let mut start = mem::size_of::(); @@ -109,12 +81,6 @@ fn deserialize_state(keyed_accounts: &mut [KeyedAccount], buffer: &[u8]) { } } -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub enum BpfLoader { - File { name: String }, - Bytes { bytes: Vec }, -} - #[no_mangle] pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -> bool { static INIT: Once = ONCE_INIT; @@ -124,43 +90,8 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) - }); if keyed_accounts[0].account.executable { - let prog: Vec; - if let Ok(program) = deserialize(&keyed_accounts[0].account.userdata) { - match program { - BpfLoader::File { name } => { - trace!("Call Bpf with file {:?}", name); - let path = create_path(&name); - let file = match elf::File::open_path(&path) { - Ok(f) => f, - Err(e) => { - warn!("Error opening ELF {:?}: {:?}", path, e); - return false; - } - }; - - let text_section = match file.get_section(PLATFORM_SECTION_RS) { - Some(s) => s, - None => match file.get_section(PLATFORM_SECTION_C) { - Some(s) => s, - None => { - warn!("Failed to find elf section {:?}", PLATFORM_SECTION_C); - return false; - } - }, - }; - prog = text_section.data.clone(); - } - BpfLoader::Bytes { bytes } => { - trace!("Call Bpf with bytes"); - prog = bytes; - } - } - } else { - warn!("deserialize failed: {:?}", tx_data); - return false; - } + let prog = keyed_accounts[0].account.userdata.clone(); trace!("Call BPF, {} Instructions", prog.len() / 8); - let vm = match create_vm(&prog) { Ok(vm) => vm, Err(e) => { @@ -168,34 +99,27 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) - return false; } }; - - let mut v = serialize_state(&mut keyed_accounts[1..], &tx_data); + let mut v = serialize_parameters(&mut keyed_accounts[1..], &tx_data); if 0 == vm.prog_exec(v.as_mut_slice()) { warn!("BPF program failed"); return false; } - deserialize_state(&mut keyed_accounts[1..], &v); + deserialize_parameters(&mut keyed_accounts[1..], &v); } else if let Ok(instruction) = deserialize(tx_data) { match instruction { LoaderInstruction::Write { offset, bytes } => { - trace!("BPFLoader::Write offset {} bytes {:?}", offset, bytes); let offset = offset as usize; - if keyed_accounts[0].account.userdata.len() <= offset + bytes.len() { + let len = bytes.len(); + trace!("LuaLoader::Write offset {} length {:?}", offset, len); + if keyed_accounts[0].account.userdata.len() < offset + len { + println!( + "Overflow {} < {}", + keyed_accounts[0].account.userdata.len(), + offset + len + ); return false; } - let name = match str::from_utf8(&bytes) { - Ok(s) => s.to_string(), - Err(e) => { - println!("Invalid UTF-8 sequence: {}", e); - return false; - } - }; - trace!("name: {:?}", name); - let s = serialize(&BpfLoader::File { name }).unwrap(); - keyed_accounts[0] - .account - .userdata - .splice(0..s.len(), s.iter().cloned()); + keyed_accounts[0].account.userdata[offset..offset + len].copy_from_slice(&bytes); } LoaderInstruction::Finalize => { keyed_accounts[0].account.executable = true; diff --git a/programs/native/lua_loader/src/lib.rs b/programs/native/lua_loader/src/lib.rs index c8a6a3c468..1fab099eeb 100644 --- a/programs/native/lua_loader/src/lib.rs +++ b/programs/native/lua_loader/src/lib.rs @@ -3,23 +3,15 @@ extern crate env_logger; #[macro_use] extern crate log; extern crate rlua; -#[macro_use] -extern crate serde_derive; extern crate solana_program_interface; -use bincode::{deserialize, serialize}; +use bincode::deserialize; use rlua::{Lua, Result, Table}; use solana_program_interface::account::KeyedAccount; use solana_program_interface::loader_instruction::LoaderInstruction; use std::str; use std::sync::{Once, ONCE_INIT}; -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub enum LuaLoader { - File { name: String }, - Bytes { bytes: Vec }, -} - /// Make KeyAccount values available to Lua. fn set_accounts(lua: &Lua, name: &str, keyed_accounts: &[KeyedAccount]) -> Result<()> { let accounts = lua.create_table()?; @@ -68,24 +60,8 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) - }); if keyed_accounts[0].account.executable { - let prog: Vec; - if let Ok(program) = deserialize(&keyed_accounts[0].account.userdata) { - match program { - LuaLoader::File { name } => { - trace!("Call Lua with file {:?}", name); - panic!("Not supported"); - } - LuaLoader::Bytes { bytes } => { - trace!("Call Lua with bytes, code size {}", bytes.len()); - prog = bytes; - } - } - } else { - warn!("deserialize failed: {:?}", tx_data); - return false; - } - - let code = str::from_utf8(&prog).unwrap(); + let code = keyed_accounts[0].account.userdata.clone(); + let code = str::from_utf8(&code).unwrap(); match run_lua(&mut keyed_accounts[1..], &code, tx_data) { Ok(()) => { trace!("Lua success"); @@ -99,17 +75,18 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) - } else if let Ok(instruction) = deserialize(tx_data) { match instruction { LoaderInstruction::Write { offset, bytes } => { - trace!("LuaLoader::Write offset {} bytes {:?}", offset, bytes); let offset = offset as usize; - if keyed_accounts[0].account.userdata.len() <= offset + bytes.len() { - warn!("program overflow offset {} len {}", offset, bytes.len()); + let len = bytes.len(); + trace!("LuaLoader::Write offset {} length {:?}", offset, len); + if keyed_accounts[0].account.userdata.len() < offset + len { + println!( + "Overflow {} < {}", + keyed_accounts[0].account.userdata.len(), + offset + len + ); return false; } - let s = serialize(&LuaLoader::Bytes { bytes }).unwrap(); - keyed_accounts[0] - .account - .userdata - .splice(0..s.len(), s.iter().cloned()); + keyed_accounts[0].account.userdata[offset..offset + len].copy_from_slice(&bytes); } LoaderInstruction::Finalize => { diff --git a/src/native_loader.rs b/src/native_loader.rs index 6ed0e88db9..fdbbf0e9a1 100644 --- a/src/native_loader.rs +++ b/src/native_loader.rs @@ -83,11 +83,11 @@ pub fn process_transaction(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) LoaderInstruction::Write { offset, bytes } => { trace!("NativeLoader::Write offset {} bytes {:?}", offset, bytes); let offset = offset as usize; - if keyed_accounts[0].account.userdata.len() <= offset + bytes.len() { + if keyed_accounts[0].account.userdata.len() < offset + bytes.len() { warn!( - "Error: Overflow, {} > {}", - offset + bytes.len(), - keyed_accounts[0].account.userdata.len() + "Error: Overflow, {} < {}", + keyed_accounts[0].account.userdata.len(), + offset + bytes.len() ); return false; } diff --git a/tests/programs.rs b/tests/programs.rs index e324412dd4..eea54ec5fa 100644 --- a/tests/programs.rs +++ b/tests/programs.rs @@ -15,9 +15,32 @@ use solana::system_transaction::SystemTransaction; use solana::tictactoe_program::Command; use solana::transaction::Transaction; use solana_program_interface::pubkey::Pubkey; +#[cfg(feature = "bpf_c")] +use std::env; +#[cfg(feature = "bpf_c")] +use std::path::PathBuf; -// TODO test modified user data -// TODO test failure if account tokens decrease but not assigned to program +/// BPF program file prefixes +#[cfg(feature = "bpf_c")] +const PLATFORM_FILE_PREFIX_BPF: &str = ""; +/// BPF program file extension +#[cfg(feature = "bpf_c")] +const PLATFORM_FILE_EXTENSION_BPF: &str = "o"; +/// BPF program ELF section name where the program code is located +pub const PLATFORM_SECTION_RS: &str = ".text,entrypoint"; +pub const PLATFORM_SECTION_C: &str = ".text.entrypoint"; +/// Create a BPF program file name +#[cfg(feature = "bpf_c")] +fn create_bpf_path(name: &str) -> PathBuf { + let pathbuf = { + let current_exe = env::current_exe().unwrap(); + PathBuf::from(current_exe.parent().unwrap().parent().unwrap()) + }; + pathbuf.join( + PathBuf::from(PLATFORM_FILE_PREFIX_BPF.to_string() + name) + .with_extension(PLATFORM_FILE_EXTENSION_BPF), + ) +} fn check_tx_results(bank: &Bank, tx: &Transaction, result: Vec>) { assert_eq!(result.len(), 1); @@ -37,14 +60,14 @@ impl Loader { let bank = Bank::new(&mint); let loader = Keypair::new(); - // allocate, populate, finalize BPF loader + // allocate, populate, finalize, and spawn BPF loader let tx = Transaction::system_create( &mint.keypair(), loader.pubkey(), mint.last_id(), 1, - 56, // TODO How does the user know how much space to allocate for what should be an internally known size + 56, // TODO native_loader::id(), 0, ); @@ -88,17 +111,17 @@ struct Program { } impl Program { - pub fn new(loader: &Loader, userdata: Vec, size: u64) -> Self { + pub fn new(loader: &Loader, userdata: Vec) -> Self { let program = Keypair::new(); - // allocate, populate, and finalize user program + // allocate, populate, and finalize and spawn program let tx = Transaction::system_create( &loader.mint.keypair(), program.pubkey(), loader.mint.last_id(), 1, - size, + userdata.len() as u64, loader.loader, 0, ); @@ -108,19 +131,24 @@ impl Program { loader.bank.process_transactions(&vec![tx.clone()]), ); - let tx = Transaction::write( - &program, - loader.loader, - 0, - userdata, - loader.mint.last_id(), - 0, - ); - check_tx_results( - &loader.bank, - &tx, - loader.bank.process_transactions(&vec![tx.clone()]), - ); + let chunk_size = 256; // Size of chunk just needs to fix into tx + let mut offset = 0; + for chunk in userdata.chunks(chunk_size) { + let tx = Transaction::write( + &program, + loader.loader, + offset, + chunk.to_vec(), + loader.mint.last_id(), + 0, + ); + check_tx_results( + &loader.bank, + &tx, + loader.bank.process_transactions(&vec![tx.clone()]), + ); + offset += chunk_size as u32; + } let tx = Transaction::finalize(&program, loader.loader, loader.mint.last_id(), 0); check_tx_results( @@ -141,18 +169,18 @@ impl Program { } #[test] -fn test_transaction_load_native() { +fn test_program_native_noop() { logger::setup(); let loader = Loader::new_native(); let name = String::from("noop"); let userdata = name.as_bytes().to_vec(); - let program = Program::new(&loader, userdata, 300); + let program = Program::new(&loader, userdata); // Call user program let tx = Transaction::new( - &loader.mint.keypair(), // TODO + &loader.mint.keypair(), &[], program.program.pubkey(), vec![1u8], @@ -178,7 +206,7 @@ fn test_program_lua_move_funds() { accounts[2].tokens = accounts[2].tokens + tokens "#.as_bytes() .to_vec(); - let program = Program::new(&loader, userdata, 300); + let program = Program::new(&loader, userdata); let from = Keypair::new(); let to = Keypair::new().pubkey(); @@ -238,14 +266,20 @@ fn test_program_bpf_noop_c() { logger::setup(); let loader = Loader::new_dynamic("bpf_loader"); - let name = String::from("noop_c"); - let userdata = name.as_bytes().to_vec(); - let program = Program::new(&loader, userdata, 56); + let program = Program::new( + &loader, + elf::File::open_path(&create_bpf_path("noop_c")) + .unwrap() + .get_section(PLATFORM_SECTION_C) + .unwrap() + .data + .clone(), + ); // Call user program let tx = Transaction::new( - &loader.mint.keypair(), // TODO + &loader.mint.keypair(), &[], program.program.pubkey(), vec![1u8], @@ -434,13 +468,19 @@ impl Dashboard { #[cfg(feature = "bpf_c")] #[test] -fn test_program_bpf_file_tictactoe_c() { +fn test_program_bpf_tictactoe_c() { logger::setup(); let loader = Loader::new_dynamic("bpf_loader"); - let name = String::from("tictactoe_c"); - let userdata = name.as_bytes().to_vec(); - let program = Program::new(&loader, userdata, 56); + let program = Program::new( + &loader, + elf::File::open_path(&create_bpf_path("tictactoe_c")) + .unwrap() + .get_section(PLATFORM_SECTION_C) + .unwrap() + .data + .clone(), + ); let player_x = Pubkey::new(&[0xA; 32]); let player_y = Pubkey::new(&[0xB; 32]); @@ -461,13 +501,19 @@ fn test_program_bpf_file_tictactoe_c() { #[cfg(feature = "bpf_c")] #[test] -fn test_program_bpf_file_tictactoe_dashboard_c() { +fn test_program_bpf_tictactoe_dashboard_c() { logger::setup(); let loader = Loader::new_dynamic("bpf_loader"); - let name = String::from("tictactoe_c"); - let userdata = name.as_bytes().to_vec(); - let ttt_program = Program::new(&loader, userdata, 56); + let ttt_program = Program::new( + &loader, + elf::File::open_path(&create_bpf_path("tictactoe_c")) + .unwrap() + .get_section(PLATFORM_SECTION_C) + .unwrap() + .data + .clone(), + ); let player_x = Pubkey::new(&[0xA; 32]); let player_y = Pubkey::new(&[0xB; 32]); @@ -494,9 +540,15 @@ fn test_program_bpf_file_tictactoe_dashboard_c() { let ttt3 = TicTacToe::new(&loader, &ttt_program); ttt3.init(&loader, &ttt_program, &player_x); - let name = String::from("tictactoe_dashboard_c"); - let userdata = name.as_bytes().to_vec(); - let dashboard_program = Program::new(&loader, userdata, 56); + let dashboard_program = Program::new( + &loader, + elf::File::open_path(&create_bpf_path("tictactoe_dashboard_c")) + .unwrap() + .get_section(PLATFORM_SECTION_C) + .unwrap() + .data + .clone(), + ); let dashboard = Dashboard::new(&loader, &dashboard_program); dashboard.update(&loader, &dashboard_program, &ttt1.id());