Load program data in chunks (#1556)

Load program data in chunks

programs/native/bpf_loader/src/lib.rs

@@ -2,51 +2,23 @@ pub mod bpf_verifier;
 
 extern crate bincode;
 extern crate byteorder;
-extern crate elf;
 extern crate env_logger;
-extern crate libc;
 #[macro_use]
 extern crate log;
 extern crate rbpf;
-#[macro_use]
-extern crate serde_derive;
 extern crate solana_program_interface;
 
-use bincode::{deserialize, serialize};
+use bincode::deserialize;
 use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
 use solana_program_interface::account::KeyedAccount;
 use solana_program_interface::loader_instruction::LoaderInstruction;
 use solana_program_interface::pubkey::Pubkey;
-use std::env;
 use std::io::prelude::*;
 use std::io::Error;
 use std::mem;
-use std::path::PathBuf;
 use std::str;
 use std::sync::{Once, ONCE_INIT};
 
-/// Dynamic link library prefixs
-const PLATFORM_FILE_PREFIX_BPF: &str = "";
-
-/// Dynamic link library file extension specific to the platform
-const PLATFORM_FILE_EXTENSION_BPF: &str = "o";
-
-/// Section name
-pub const PLATFORM_SECTION_RS: &str = ".text,entrypoint";
-pub const PLATFORM_SECTION_C: &str = ".text.entrypoint";
-
-fn create_path(name: &str) -> PathBuf {
-    let pathbuf = {
-        let current_exe = env::current_exe().unwrap();
-        PathBuf::from(current_exe.parent().unwrap().parent().unwrap())
-    };
-
-    pathbuf.join(
-        PathBuf::from(PLATFORM_FILE_PREFIX_BPF.to_string() + name)
-            .with_extension(PLATFORM_FILE_EXTENSION_BPF),
-    )
-}
-
 fn create_vm(prog: &[u8]) -> Result<rbpf::EbpfVmRaw, Error> {
     let mut vm = rbpf::EbpfVmRaw::new(None)?;
     vm.set_verifier(bpf_verifier::check)?;
@@ -72,7 +44,7 @@ fn dump_prog(name: &str, prog: &[u8]) {
     }
 }
 
-fn serialize_state(keyed_accounts: &mut [KeyedAccount], data: &[u8]) -> Vec<u8> {
+fn serialize_parameters(keyed_accounts: &mut [KeyedAccount], data: &[u8]) -> Vec<u8> {
     assert_eq!(32, mem::size_of::<Pubkey>());
 
     let mut v: Vec<u8> = Vec::new();
@@ -91,7 +63,7 @@ fn serialize_state(keyed_accounts: &mut [KeyedAccount], data: &[u8]) -> Vec<u8>
     v
 }
 
-fn deserialize_state(keyed_accounts: &mut [KeyedAccount], buffer: &[u8]) {
+fn deserialize_parameters(keyed_accounts: &mut [KeyedAccount], buffer: &[u8]) {
     assert_eq!(32, mem::size_of::<Pubkey>());
 
     let mut start = mem::size_of::<u64>();
@@ -109,12 +81,6 @@ fn deserialize_state(keyed_accounts: &mut [KeyedAccount], buffer: &[u8]) {
     }
 }
 
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
-pub enum BpfLoader {
-    File { name: String },
-    Bytes { bytes: Vec<u8> },
-}
-
 #[no_mangle]
 pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -> bool {
     static INIT: Once = ONCE_INIT;
@@ -124,43 +90,8 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -
     });
 
     if keyed_accounts[0].account.executable {
-        let prog: Vec<u8>;
+        let prog = keyed_accounts[0].account.userdata.clone();
-        if let Ok(program) = deserialize(&keyed_accounts[0].account.userdata) {
-            match program {
-                BpfLoader::File { name } => {
-                    trace!("Call Bpf with file {:?}", name);
-                    let path = create_path(&name);
-                    let file = match elf::File::open_path(&path) {
-                        Ok(f) => f,
-                        Err(e) => {
-                            warn!("Error opening ELF {:?}: {:?}", path, e);
-                            return false;
-                        }
-                    };
-
-                    let text_section = match file.get_section(PLATFORM_SECTION_RS) {
-                        Some(s) => s,
-                        None => match file.get_section(PLATFORM_SECTION_C) {
-                            Some(s) => s,
-                            None => {
-                                warn!("Failed to find elf section {:?}", PLATFORM_SECTION_C);
-                                return false;
-                            }
-                        },
-                    };
-                    prog = text_section.data.clone();
-                }
-                BpfLoader::Bytes { bytes } => {
-                    trace!("Call Bpf with bytes");
-                    prog = bytes;
-                }
-            }
-        } else {
-            warn!("deserialize failed: {:?}", tx_data);
-            return false;
-        }
         trace!("Call BPF, {} Instructions", prog.len() / 8);
-
         let vm = match create_vm(&prog) {
             Ok(vm) => vm,
             Err(e) => {
@@ -168,34 +99,27 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -
                 return false;
             }
         };
-
+        let mut v = serialize_parameters(&mut keyed_accounts[1..], &tx_data);
-        let mut v = serialize_state(&mut keyed_accounts[1..], &tx_data);
         if 0 == vm.prog_exec(v.as_mut_slice()) {
             warn!("BPF program failed");
             return false;
         }
-        deserialize_state(&mut keyed_accounts[1..], &v);
+        deserialize_parameters(&mut keyed_accounts[1..], &v);
     } else if let Ok(instruction) = deserialize(tx_data) {
         match instruction {
             LoaderInstruction::Write { offset, bytes } => {
-                trace!("BPFLoader::Write offset {} bytes {:?}", offset, bytes);
                 let offset = offset as usize;
-                if keyed_accounts[0].account.userdata.len() <= offset + bytes.len() {
+                let len = bytes.len();
+                trace!("LuaLoader::Write offset {} length {:?}", offset, len);
+                if keyed_accounts[0].account.userdata.len() < offset + len {
+                    println!(
+                        "Overflow {} < {}",
+                        keyed_accounts[0].account.userdata.len(),
+                        offset + len
+                    );
                     return false;
                 }
-                let name = match str::from_utf8(&bytes) {
+                keyed_accounts[0].account.userdata[offset..offset + len].copy_from_slice(&bytes);
-                    Ok(s) => s.to_string(),
-                    Err(e) => {
-                        println!("Invalid UTF-8 sequence: {}", e);
-                        return false;
-                    }
-                };
-                trace!("name: {:?}", name);
-                let s = serialize(&BpfLoader::File { name }).unwrap();
-                keyed_accounts[0]
-                    .account
-                    .userdata
-                    .splice(0..s.len(), s.iter().cloned());
             }
             LoaderInstruction::Finalize => {
                 keyed_accounts[0].account.executable = true;

programs/native/lua_loader/src/lib.rs

@@ -3,23 +3,15 @@ extern crate env_logger;
 #[macro_use]
 extern crate log;
 extern crate rlua;
-#[macro_use]
-extern crate serde_derive;
 extern crate solana_program_interface;
 
-use bincode::{deserialize, serialize};
+use bincode::deserialize;
 use rlua::{Lua, Result, Table};
 use solana_program_interface::account::KeyedAccount;
 use solana_program_interface::loader_instruction::LoaderInstruction;
 use std::str;
 use std::sync::{Once, ONCE_INIT};
 
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
-pub enum LuaLoader {
-    File { name: String },
-    Bytes { bytes: Vec<u8> },
-}
-
 /// Make KeyAccount values available to Lua.
 fn set_accounts(lua: &Lua, name: &str, keyed_accounts: &[KeyedAccount]) -> Result<()> {
     let accounts = lua.create_table()?;
@@ -68,24 +60,8 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -
     });
 
     if keyed_accounts[0].account.executable {
-        let prog: Vec<u8>;
+        let code = keyed_accounts[0].account.userdata.clone();
-        if let Ok(program) = deserialize(&keyed_accounts[0].account.userdata) {
+        let code = str::from_utf8(&code).unwrap();
-            match program {
-                LuaLoader::File { name } => {
-                    trace!("Call Lua with file {:?}", name);
-                    panic!("Not supported");
-                }
-                LuaLoader::Bytes { bytes } => {
-                    trace!("Call Lua with bytes, code size {}", bytes.len());
-                    prog = bytes;
-                }
-            }
-        } else {
-            warn!("deserialize failed: {:?}", tx_data);
-            return false;
-        }
-
-        let code = str::from_utf8(&prog).unwrap();
         match run_lua(&mut keyed_accounts[1..], &code, tx_data) {
             Ok(()) => {
                 trace!("Lua success");
@@ -99,17 +75,18 @@ pub extern "C" fn process(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8]) -
     } else if let Ok(instruction) = deserialize(tx_data) {
         match instruction {
             LoaderInstruction::Write { offset, bytes } => {
-                trace!("LuaLoader::Write offset {} bytes {:?}", offset, bytes);
                 let offset = offset as usize;
-                if keyed_accounts[0].account.userdata.len() <= offset + bytes.len() {
+                let len = bytes.len();
-                    warn!("program overflow offset {} len {}", offset, bytes.len());
+                trace!("LuaLoader::Write offset {} length {:?}", offset, len);
+                if keyed_accounts[0].account.userdata.len() < offset + len {
+                    println!(
+                        "Overflow {} < {}",
+                        keyed_accounts[0].account.userdata.len(),
+                        offset + len
+                    );
                     return false;
                 }
-                let s = serialize(&LuaLoader::Bytes { bytes }).unwrap();
+                keyed_accounts[0].account.userdata[offset..offset + len].copy_from_slice(&bytes);
-                keyed_accounts[0]
-                    .account
-                    .userdata
-                    .splice(0..s.len(), s.iter().cloned());
             }
 
             LoaderInstruction::Finalize => {

src/native_loader.rs

@@ -83,11 +83,11 @@ pub fn process_transaction(keyed_accounts: &mut [KeyedAccount], tx_data: &[u8])
             LoaderInstruction::Write { offset, bytes } => {
                 trace!("NativeLoader::Write offset {} bytes {:?}", offset, bytes);
                 let offset = offset as usize;
-                if keyed_accounts[0].account.userdata.len() <= offset + bytes.len() {
+                if keyed_accounts[0].account.userdata.len() < offset + bytes.len() {
                     warn!(
-                        "Error: Overflow, {} > {}",
+                        "Error: Overflow, {} < {}",
-                        offset + bytes.len(),
+                        keyed_accounts[0].account.userdata.len(),
-                        keyed_accounts[0].account.userdata.len()
+                        offset + bytes.len()
                     );
                     return false;
                 }

tests/programs.rs

@@ -15,9 +15,32 @@ use solana::system_transaction::SystemTransaction;
 use solana::tictactoe_program::Command;
 use solana::transaction::Transaction;
 use solana_program_interface::pubkey::Pubkey;
+#[cfg(feature = "bpf_c")]
+use std::env;
+#[cfg(feature = "bpf_c")]
+use std::path::PathBuf;
 
-// TODO test modified user data
+/// BPF program file prefixes
-// TODO test failure if account tokens decrease but not assigned to program
+#[cfg(feature = "bpf_c")]
+const PLATFORM_FILE_PREFIX_BPF: &str = "";
+/// BPF program file extension
+#[cfg(feature = "bpf_c")]
+const PLATFORM_FILE_EXTENSION_BPF: &str = "o";
+/// BPF program ELF section name where the program code is located
+pub const PLATFORM_SECTION_RS: &str = ".text,entrypoint";
+pub const PLATFORM_SECTION_C: &str = ".text.entrypoint";
+/// Create a BPF program file name
+#[cfg(feature = "bpf_c")]
+fn create_bpf_path(name: &str) -> PathBuf {
+    let pathbuf = {
+        let current_exe = env::current_exe().unwrap();
+        PathBuf::from(current_exe.parent().unwrap().parent().unwrap())
+    };
+    pathbuf.join(
+        PathBuf::from(PLATFORM_FILE_PREFIX_BPF.to_string() + name)
+            .with_extension(PLATFORM_FILE_EXTENSION_BPF),
+    )
+}
 
 fn check_tx_results(bank: &Bank, tx: &Transaction, result: Vec<solana::bank::Result<()>>) {
     assert_eq!(result.len(), 1);
@@ -37,14 +60,14 @@ impl Loader {
         let bank = Bank::new(&mint);
         let loader = Keypair::new();
 
-        // allocate, populate, finalize BPF loader
+        // allocate, populate, finalize, and spawn BPF loader
 
         let tx = Transaction::system_create(
             &mint.keypair(),
             loader.pubkey(),
             mint.last_id(),
             1,
-            56, // TODO How does the user know how much space to allocate for what should be an internally known size
+            56, // TODO
             native_loader::id(),
             0,
         );
@@ -88,17 +111,17 @@ struct Program {
 }
 
 impl Program {
-    pub fn new(loader: &Loader, userdata: Vec<u8>, size: u64) -> Self {
+    pub fn new(loader: &Loader, userdata: Vec<u8>) -> Self {
         let program = Keypair::new();
 
-        // allocate, populate, and finalize user program
+        // allocate, populate, and finalize and spawn program
 
         let tx = Transaction::system_create(
             &loader.mint.keypair(),
             program.pubkey(),
             loader.mint.last_id(),
             1,
-            size,
+            userdata.len() as u64,
             loader.loader,
             0,
         );
@@ -108,11 +131,14 @@ impl Program {
             loader.bank.process_transactions(&vec![tx.clone()]),
         );
 
+        let chunk_size = 256; // Size of chunk just needs to fix into tx
+        let mut offset = 0;
+        for chunk in userdata.chunks(chunk_size) {
             let tx = Transaction::write(
                 &program,
                 loader.loader,
-            0,
+                offset,
-            userdata,
+                chunk.to_vec(),
                 loader.mint.last_id(),
                 0,
             );
@@ -121,6 +147,8 @@ impl Program {
                 &tx,
                 loader.bank.process_transactions(&vec![tx.clone()]),
             );
+            offset += chunk_size as u32;
+        }
 
         let tx = Transaction::finalize(&program, loader.loader, loader.mint.last_id(), 0);
         check_tx_results(
@@ -141,18 +169,18 @@ impl Program {
 }
 
 #[test]
-fn test_transaction_load_native() {
+fn test_program_native_noop() {
     logger::setup();
 
     let loader = Loader::new_native();
     let name = String::from("noop");
     let userdata = name.as_bytes().to_vec();
-    let program = Program::new(&loader, userdata, 300);
+    let program = Program::new(&loader, userdata);
 
     // Call user program
 
     let tx = Transaction::new(
-        &loader.mint.keypair(), // TODO
+        &loader.mint.keypair(),
         &[],
         program.program.pubkey(),
         vec![1u8],
@@ -178,7 +206,7 @@ fn test_program_lua_move_funds() {
             accounts[2].tokens = accounts[2].tokens + tokens
         "#.as_bytes()
     .to_vec();
-    let program = Program::new(&loader, userdata, 300);
+    let program = Program::new(&loader, userdata);
     let from = Keypair::new();
     let to = Keypair::new().pubkey();
 
@@ -238,14 +266,20 @@ fn test_program_bpf_noop_c() {
     logger::setup();
 
     let loader = Loader::new_dynamic("bpf_loader");
-    let name = String::from("noop_c");
+    let program = Program::new(
-    let userdata = name.as_bytes().to_vec();
+        &loader,
-    let program = Program::new(&loader, userdata, 56);
+        elf::File::open_path(&create_bpf_path("noop_c"))
+            .unwrap()
+            .get_section(PLATFORM_SECTION_C)
+            .unwrap()
+            .data
+            .clone(),
+    );
 
     // Call user program
 
     let tx = Transaction::new(
-        &loader.mint.keypair(), // TODO
+        &loader.mint.keypair(),
         &[],
         program.program.pubkey(),
         vec![1u8],
@@ -434,13 +468,19 @@ impl Dashboard {
 
 #[cfg(feature = "bpf_c")]
 #[test]
-fn test_program_bpf_file_tictactoe_c() {
+fn test_program_bpf_tictactoe_c() {
     logger::setup();
 
     let loader = Loader::new_dynamic("bpf_loader");
-    let name = String::from("tictactoe_c");
+    let program = Program::new(
-    let userdata = name.as_bytes().to_vec();
+        &loader,
-    let program = Program::new(&loader, userdata, 56);
+        elf::File::open_path(&create_bpf_path("tictactoe_c"))
+            .unwrap()
+            .get_section(PLATFORM_SECTION_C)
+            .unwrap()
+            .data
+            .clone(),
+    );
     let player_x = Pubkey::new(&[0xA; 32]);
     let player_y = Pubkey::new(&[0xB; 32]);
 
@@ -461,13 +501,19 @@ fn test_program_bpf_file_tictactoe_c() {
 
 #[cfg(feature = "bpf_c")]
 #[test]
-fn test_program_bpf_file_tictactoe_dashboard_c() {
+fn test_program_bpf_tictactoe_dashboard_c() {
     logger::setup();
 
     let loader = Loader::new_dynamic("bpf_loader");
-    let name = String::from("tictactoe_c");
+    let ttt_program = Program::new(
-    let userdata = name.as_bytes().to_vec();
+        &loader,
-    let ttt_program = Program::new(&loader, userdata, 56);
+        elf::File::open_path(&create_bpf_path("tictactoe_c"))
+            .unwrap()
+            .get_section(PLATFORM_SECTION_C)
+            .unwrap()
+            .data
+            .clone(),
+    );
     let player_x = Pubkey::new(&[0xA; 32]);
     let player_y = Pubkey::new(&[0xB; 32]);
 
@@ -494,9 +540,15 @@ fn test_program_bpf_file_tictactoe_dashboard_c() {
     let ttt3 = TicTacToe::new(&loader, &ttt_program);
     ttt3.init(&loader, &ttt_program, &player_x);
 
-    let name = String::from("tictactoe_dashboard_c");
+    let dashboard_program = Program::new(
-    let userdata = name.as_bytes().to_vec();
+        &loader,
-    let dashboard_program = Program::new(&loader, userdata, 56);
+        elf::File::open_path(&create_bpf_path("tictactoe_dashboard_c"))
+            .unwrap()
+            .get_section(PLATFORM_SECTION_C)
+            .unwrap()
+            .data
+            .clone(),
+    );
     let dashboard = Dashboard::new(&loader, &dashboard_program);
 
     dashboard.update(&loader, &dashboard_program, &ttt1.id());