blockworks-foundation
/
solana-with-rpc-optimizations
mirror of https://github.com/blockworks-foundation/solana-with-rpc-optimizations.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Load program data in chunks (#1556) 

Load program data in chunks
This commit is contained in:
jackcmay 2018-10-19 18:28:38 -07:00 committed by GitHub
parent dfde83bdce
commit f8516b677a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 122 additions and 169 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
programs/native/bpf_loader/src/lib.rs
View File

@ -2,51 +2,23 @@ pub mod bpf_verifier;
extern crate bincode;
extern crate byteorder;
extern crate elf;
extern crate env_logger;
extern crate libc;
#[macro_use]
extern crate log;
extern crate rbpf;
#[macro_use]
extern crate serde_derive;
extern crate solana_program_interface;
use bincode::{deserialize, serialize};
use bincode::deserialize;
use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
use solana_program_interface::account::KeyedAccount;
use solana_program_interface::loader_instruction::LoaderInstruction;
use solana_program_interface::pubkey::Pubkey;
use std::env;
use std::io::prelude::*;
use std::io::Error;
use std::mem;
use std::path::PathBuf;
use std::str;
use std::sync::{Once, ONCE_INIT};
/// Dynamic link library prefixs
const PLATFORM_FILE_PREFIX_BPF: &str = "";
/// Dynamic link library file extension specific to the platform
const PLATFORM_FILE_EXTENSION_BPF: &str = "o";
/// Section name
pub const PLATFORM_SECTION_RS: &str = ".text,entrypoint";
pub const PLATFORM_SECTION_C: &str = ".text.entrypoint";
fn create_path(name: &str) -> PathBuf {
let pathbuf = {
let current_exe = env::current_exe().unwrap();
PathBuf::from(current_exe.parent().unwrap().parent().unwrap())
};
pathbuf.join(
PathBuf::from(PLATFORM_FILE_PREFIX_BPF.to_string() + name)
.with_extension(PLATFORM_FILE_EXTENSION_BPF),
)
}
fn create_vm(prog: &[u8]) -> Result<rbpf::EbpfVmRaw, Error> {
let mut vm = rbpf::EbpfVmRaw::new(None)?;
vm.set_verifier(bpf_verifier::check)?;
@ -72,7 +44,7 @@ fn dump_prog(name: &str, prog: &[u8]) {
}
}
fn serialize_state(keyed_accounts: &mut [KeyedAccount], data: &[u8]) -> Vec<u8> {
fn serialize_parameters(keyed_accounts: &mut [KeyedAccount], data: &[u8]) -> Vec<u8> {
assert_eq!(32, mem::size_of::<Pubkey>());
let mut v: Vec<u8> = Vec::new();
@ -91,7 +63,7 @@ fn serialize_state(keyed_accounts: &mut [KeyedAccount], data: &[u8]) -> Vec<u8>
v
}
fn deserialize_state(keyed_accounts: &mut [KeyedAccount], buffer: &[u8]) {
fn deserialize_parameters(keyed_accounts: &mut [KeyedAccount], buffer: &[u8]) {
assert_eq!(32, mem::size_of::<Pubkey>());
let mut start = mem::size_of::<u64>();
@ -109,12 +81,6 @@ fn deserialize_state(keyed_accounts: &mut [KeyedAccount], buffer: &[u8]) {
}
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
pub enum BpfLoader {
File { name: String },
Bytes { bytes: Vec<u8> },
}
#[no_mangle]
pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -> bool {
static INIT: Once = ONCE_INIT;
@ -124,43 +90,8 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -
});
if keyed_accounts[0].account.executable {
let prog: Vec<u8>;
if let Ok(program) = deserialize(&keyed_accounts[0].account.userdata) {
match program {
BpfLoader::File { name } => {
trace!("Call Bpf with file {:?}", name);
let path = create_path(&name);
let file = match elf::File::open_path(&path) {
Ok(f) => f,
Err(e) => {
warn!("Error opening ELF {:?}: {:?}", path, e);
return false;
}
};
let text_section = match file.get_section(PLATFORM_SECTION_RS) {
Some(s) => s,
None => match file.get_section(PLATFORM_SECTION_C) {
Some(s) => s,
None => {
warn!("Failed to find elf section {:?}", PLATFORM_SECTION_C);
return false;
}
},
};
prog = text_section.data.clone();
}
BpfLoader::Bytes { bytes } => {
trace!("Call Bpf with bytes");
prog = bytes;
}
}
} else {
warn!("deserialize failed: {:?}", tx_data);
return false;
}
let prog = keyed_accounts[0].account.userdata.clone();
trace!("Call BPF, {} Instructions", prog.len() / 8);
let vm = match create_vm(&prog) {
Ok(vm) => vm,
Err(e) => {
@ -168,34 +99,27 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -
return false;
}
};
let mut v = serialize_state(&mut keyed_accounts[1..], &tx_data);
let mut v = serialize_parameters(&mut keyed_accounts[1..], &tx_data);
if 0 == vm.prog_exec(v.as_mut_slice()) {
warn!("BPF program failed");
return false;
}
deserialize_state(&mut keyed_accounts[1..], &v);
deserialize_parameters(&mut keyed_accounts[1..], &v);
} else if let Ok(instruction) = deserialize(tx_data) {
match instruction {
LoaderInstruction::Write { offset, bytes } => {
trace!("BPFLoader::Write offset {} bytes {:?}", offset, bytes);
let offset = offset as usize;
if keyed_accounts[0].account.userdata.len() <= offset + bytes.len() {
let len = bytes.len();
trace!("LuaLoader::Write offset {} length {:?}", offset, len);
if keyed_accounts[0].account.userdata.len() < offset + len {
println!(
"Overflow {} < {}",
keyed_accounts[0].account.userdata.len(),
offset + len
);
return false;
}
let name = match str::from_utf8(&bytes) {
Ok(s) => s.to_string(),
Err(e) => {
println!("Invalid UTF-8 sequence: {}", e);
return false;
}
};
trace!("name: {:?}", name);
let s = serialize(&BpfLoader::File { name }).unwrap();
keyed_accounts[0]
.account
.userdata
.splice(0..s.len(), s.iter().cloned());
keyed_accounts[0].account.userdata[offset..offset + len].copy_from_slice(&bytes);
}
LoaderInstruction::Finalize => {
keyed_accounts[0].account.executable = true;

47
programs/native/lua_loader/src/lib.rs
View File

@ -3,23 +3,15 @@ extern crate env_logger;
#[macro_use]
extern crate log;
extern crate rlua;
#[macro_use]
extern crate serde_derive;
extern crate solana_program_interface;
use bincode::{deserialize, serialize};
use bincode::deserialize;
use rlua::{Lua, Result, Table};
use solana_program_interface::account::KeyedAccount;
use solana_program_interface::loader_instruction::LoaderInstruction;
use std::str;
use std::sync::{Once, ONCE_INIT};
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
pub enum LuaLoader {
File { name: String },
Bytes { bytes: Vec<u8> },
}
/// Make KeyAccount values available to Lua.
fn set_accounts(lua: &Lua, name: &str, keyed_accounts: &[KeyedAccount]) -> Result<()> {
let accounts = lua.create_table()?;
@ -68,24 +60,8 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -
});
if keyed_accounts[0].account.executable {
let prog: Vec<u8>;
if let Ok(program) = deserialize(&keyed_accounts[0].account.userdata) {
match program {
LuaLoader::File { name } => {
trace!("Call Lua with file {:?}", name);
panic!("Not supported");
}
LuaLoader::Bytes { bytes } => {
trace!("Call Lua with bytes, code size {}", bytes.len());
prog = bytes;
}
}
} else {
warn!("deserialize failed: {:?}", tx_data);
return false;
}
let code = str::from_utf8(&prog).unwrap();
let code = keyed_accounts[0].account.userdata.clone();
let code = str::from_utf8(&code).unwrap();
match run_lua(&mut keyed_accounts[1..], &code, tx_data) {
Ok(()) => {
trace!("Lua success");
@ -99,17 +75,18 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -
} else if let Ok(instruction) = deserialize(tx_data) {
match instruction {
LoaderInstruction::Write { offset, bytes } => {
trace!("LuaLoader::Write offset {} bytes {:?}", offset, bytes);
let offset = offset as usize;
if keyed_accounts[0].account.userdata.len() <= offset + bytes.len() {
warn!("program overflow offset {} len {}", offset, bytes.len());
let len = bytes.len();
trace!("LuaLoader::Write offset {} length {:?}", offset, len);
if keyed_accounts[0].account.userdata.len() < offset + len {
println!(
"Overflow {} < {}",
keyed_accounts[0].account.userdata.len(),
offset + len
);
return false;
}
let s = serialize(&LuaLoader::Bytes { bytes }).unwrap();
keyed_accounts[0]
.account
.userdata
.splice(0..s.len(), s.iter().cloned());
keyed_accounts[0].account.userdata[offset..offset + len].copy_from_slice(&bytes);
}
LoaderInstruction::Finalize => {

8
src/native_loader.rs
View File

@ -83,11 +83,11 @@ pub fn process_transaction(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8])
LoaderInstruction::Write { offset, bytes } => {
trace!("NativeLoader::Write offset {} bytes {:?}", offset, bytes);
let offset = offset as usize;
if keyed_accounts[0].account.userdata.len() <= offset + bytes.len() {
if keyed_accounts[0].account.userdata.len() < offset + bytes.len() {
warn!(
"Error: Overflow, {} > {}",
offset + bytes.len(),
keyed_accounts[0].account.userdata.len()
"Error: Overflow, {} < {}",
keyed_accounts[0].account.userdata.len(),
offset + bytes.len()
);
return false;
}

130
tests/programs.rs
View File

@ -15,9 +15,32 @@ use solana::system_transaction::SystemTransaction;
use solana::tictactoe_program::Command;
use solana::transaction::Transaction;
use solana_program_interface::pubkey::Pubkey;
#[cfg(feature = "bpf_c")]
use std::env;
#[cfg(feature = "bpf_c")]
use std::path::PathBuf;
// TODO test modified user data
// TODO test failure if account tokens decrease but not assigned to program
/// BPF program file prefixes
#[cfg(feature = "bpf_c")]
const PLATFORM_FILE_PREFIX_BPF: &str = "";
/// BPF program file extension
#[cfg(feature = "bpf_c")]
const PLATFORM_FILE_EXTENSION_BPF: &str = "o";
/// BPF program ELF section name where the program code is located
pub const PLATFORM_SECTION_RS: &str = ".text,entrypoint";
pub const PLATFORM_SECTION_C: &str = ".text.entrypoint";
/// Create a BPF program file name
#[cfg(feature = "bpf_c")]
fn create_bpf_path(name: &str) -> PathBuf {
let pathbuf = {
let current_exe = env::current_exe().unwrap();
PathBuf::from(current_exe.parent().unwrap().parent().unwrap())
};
pathbuf.join(
PathBuf::from(PLATFORM_FILE_PREFIX_BPF.to_string() + name)
.with_extension(PLATFORM_FILE_EXTENSION_BPF),
)
}
fn check_tx_results(bank: &Bank, tx: &Transaction, result: Vec<solana::bank::Result<()>>) {
assert_eq!(result.len(), 1);
@ -37,14 +60,14 @@ impl Loader {
let bank = Bank::new(&mint);
let loader = Keypair::new();
// allocate, populate, finalize BPF loader
// allocate, populate, finalize, and spawn BPF loader
let tx = Transaction::system_create(
&mint.keypair(),
loader.pubkey(),
mint.last_id(),
1,
56, // TODO How does the user know how much space to allocate for what should be an internally known size
56, // TODO
native_loader::id(),
0,
);
@ -88,17 +111,17 @@ struct Program {
}
impl Program {
pub fn new(loader: &Loader, userdata: Vec<u8>, size: u64) -> Self {
pub fn new(loader: &Loader, userdata: Vec<u8>) -> Self {
let program = Keypair::new();
// allocate, populate, and finalize user program
// allocate, populate, and finalize and spawn program
let tx = Transaction::system_create(
&loader.mint.keypair(),
program.pubkey(),
loader.mint.last_id(),
1,
size,
userdata.len() as u64,
loader.loader,
0,
);
@ -108,19 +131,24 @@ impl Program {
loader.bank.process_transactions(&vec![tx.clone()]),
);
let tx = Transaction::write(
&program,
loader.loader,
0,
userdata,
loader.mint.last_id(),
0,
);
check_tx_results(
&loader.bank,
&tx,
loader.bank.process_transactions(&vec![tx.clone()]),
);
let chunk_size = 256; // Size of chunk just needs to fix into tx
let mut offset = 0;
for chunk in userdata.chunks(chunk_size) {
let tx = Transaction::write(
&program,
loader.loader,
offset,
chunk.to_vec(),
loader.mint.last_id(),
0,
);
check_tx_results(
&loader.bank,
&tx,
loader.bank.process_transactions(&vec![tx.clone()]),
);
offset += chunk_size as u32;
}
let tx = Transaction::finalize(&program, loader.loader, loader.mint.last_id(), 0);
check_tx_results(
@ -141,18 +169,18 @@ impl Program {
}
#[test]
fn test_transaction_load_native() {
fn test_program_native_noop() {
logger::setup();
let loader = Loader::new_native();
let name = String::from("noop");
let userdata = name.as_bytes().to_vec();
let program = Program::new(&loader, userdata, 300);
let program = Program::new(&loader, userdata);
// Call user program
let tx = Transaction::new(
&loader.mint.keypair(), // TODO
&loader.mint.keypair(),
&[],
program.program.pubkey(),
vec![1u8],
@ -178,7 +206,7 @@ fn test_program_lua_move_funds() {
accounts[2].tokens = accounts[2].tokens + tokens
"#.as_bytes()
.to_vec();
let program = Program::new(&loader, userdata, 300);
let program = Program::new(&loader, userdata);
let from = Keypair::new();
let to = Keypair::new().pubkey();
@ -238,14 +266,20 @@ fn test_program_bpf_noop_c() {
logger::setup();
let loader = Loader::new_dynamic("bpf_loader");
let name = String::from("noop_c");
let userdata = name.as_bytes().to_vec();
let program = Program::new(&loader, userdata, 56);
let program = Program::new(
&loader,
elf::File::open_path(&create_bpf_path("noop_c"))
.unwrap()
.get_section(PLATFORM_SECTION_C)
.unwrap()
.data
.clone(),
);
// Call user program
let tx = Transaction::new(
&loader.mint.keypair(), // TODO
&loader.mint.keypair(),
&[],
program.program.pubkey(),
vec![1u8],
@ -434,13 +468,19 @@ impl Dashboard {
#[cfg(feature = "bpf_c")]
#[test]
fn test_program_bpf_file_tictactoe_c() {
fn test_program_bpf_tictactoe_c() {
logger::setup();
let loader = Loader::new_dynamic("bpf_loader");
let name = String::from("tictactoe_c");
let userdata = name.as_bytes().to_vec();
let program = Program::new(&loader, userdata, 56);
let program = Program::new(
&loader,
elf::File::open_path(&create_bpf_path("tictactoe_c"))
.unwrap()
.get_section(PLATFORM_SECTION_C)
.unwrap()
.data
.clone(),
);
let player_x = Pubkey::new(&[0xA; 32]);
let player_y = Pubkey::new(&[0xB; 32]);
@ -461,13 +501,19 @@ fn test_program_bpf_file_tictactoe_c() {
#[cfg(feature = "bpf_c")]
#[test]
fn test_program_bpf_file_tictactoe_dashboard_c() {
fn test_program_bpf_tictactoe_dashboard_c() {
logger::setup();
let loader = Loader::new_dynamic("bpf_loader");
let name = String::from("tictactoe_c");
let userdata = name.as_bytes().to_vec();
let ttt_program = Program::new(&loader, userdata, 56);
let ttt_program = Program::new(
&loader,
elf::File::open_path(&create_bpf_path("tictactoe_c"))
.unwrap()
.get_section(PLATFORM_SECTION_C)
.unwrap()
.data
.clone(),
);
let player_x = Pubkey::new(&[0xA; 32]);
let player_y = Pubkey::new(&[0xB; 32]);
@ -494,9 +540,15 @@ fn test_program_bpf_file_tictactoe_dashboard_c() {
let ttt3 = TicTacToe::new(&loader, &ttt_program);
ttt3.init(&loader, &ttt_program, &player_x);
let name = String::from("tictactoe_dashboard_c");
let userdata = name.as_bytes().to_vec();
let dashboard_program = Program::new(&loader, userdata, 56);
let dashboard_program = Program::new(
&loader,
elf::File::open_path(&create_bpf_path("tictactoe_dashboard_c"))
.unwrap()
.get_section(PLATFORM_SECTION_C)
.unwrap()
.data
.clone(),
);
let dashboard = Dashboard::new(&loader, &dashboard_program);
dashboard.update(&loader, &dashboard_program, &ttt1.id());