Embed Move (#5150)

automerge
This commit is contained in:
Greg Fitzgerald 2019-07-22 13:01:52 -06:00 committed by Grimes
parent 2d42c1e33e
commit 186b514ebb
14 changed files with 4226 additions and 2 deletions

View File

@ -48,4 +48,9 @@ members = [
"vote-signer",
"wallet",
]
exclude = ["programs/bpf/rust/noop"]
exclude = [
"programs/bpf/rust/noop",
"programs/move_loader_api",
"programs/move_loader_program",
]

View File

@ -15,6 +15,9 @@ steps:
- command: "ci/test-bench.sh"
name: "bench"
timeout_in_minutes: 60
- command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-move-demo.sh"
name: "move-demo"
timeout_in_minutes: 30
- command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh"
name: "stable"
timeout_in_minutes: 40

View File

@ -2,6 +2,10 @@
# ci/rust-version.sh to pick up the new image tag
FROM rust:1.36.0
# Add Google Protocol Buffers for Libra's metrics library.
ENV PROTOC_VERSION 3.8.0
ENV PROTOC_ZIP protoc-$PROTOC_VERSION-linux-x86_64.zip
RUN set -x \
&& apt update \
&& apt-get install apt-transport-https \
@ -20,6 +24,8 @@ RUN set -x \
mscgen \
rsync \
sudo \
golang \
unzip \
\
&& rm -rf /var/lib/apt/lists/* \
&& rustup component add rustfmt \
@ -28,4 +34,8 @@ RUN set -x \
&& cargo install svgbob_cli \
&& cargo install mdbook \
&& rustc --version \
&& cargo --version
&& cargo --version \
&& curl -OL https://github.com/google/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP \
&& unzip -o $PROTOC_ZIP -d /usr/local bin/protoc \
&& unzip -o $PROTOC_ZIP -d /usr/local include/* \
&& rm -f $PROTOC_ZIP

33
ci/test-move-demo.sh Executable file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env bash
set -e
cd "$(dirname "$0")/.."
annotate() {
${BUILDKITE:-false} && {
buildkite-agent annotate "$@"
}
}
ci/affects-files.sh \
.rs$ \
Cargo.lock$ \
Cargo.toml$ \
ci/test-move-demo.sh \
|| {
annotate --style info --context test-bench \
"Bench skipped as no .rs files were modified"
exit 0
}
source ci/_
source ci/upload-ci-artifact.sh
eval "$(ci/channel-info.sh)"
source ci/rust-version.sh nightly
set -o pipefail
export RUST_BACKTRACE=1
# Run Move tests
_ cargo +"$rust_nightly" test --manifest-path=programs/move_loader_program/Cargo.toml ${V:+--verbose}

2
programs/move_loader_api/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
/farf/
/target/

View File

@ -0,0 +1,39 @@
[package]
name = "solana-move-loader-api"
version = "0.17.0"
description = "Solana Move Loader"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
edition = "2018"
[dependencies]
bincode = "1.1.4"
byteorder = "1.3.2"
libc = "0.2.58"
log = "0.4.2"
serde = "1.0.94"
serde_derive = "1.0.94"
serde_json = "1.0.40"
solana-logger = { path = "../../logger", version = "0.17.0" }
solana-sdk = { path = "../../sdk", version = "0.17.0" }
bytecode_verifier = { git = "https://github.com/solana-labs/libra", tag = "v0.0.0.0" }
failure = { git = "https://github.com/solana-labs/libra", tag = "v0.0.0.0", package = "failure_ext" }
language_e2e_tests = { git = "https://github.com/solana-labs/libra", tag = "v0.0.0.0" }
lazy_static = "1.3.0"
protobuf = "2.7"
proto_conv = { git = "https://github.com/solana-labs/libra", tag = "v0.0.0.0" }
state_view = { git = "https://github.com/solana-labs/libra", tag = "v0.0.0.0" }
types = { git = "https://github.com/solana-labs/libra", tag = "v0.0.0.0" }
vm = { git = "https://github.com/solana-labs/libra", tag = "v0.0.0.0" }
vm_cache_map = { git = "https://github.com/solana-labs/libra", tag = "v0.0.0.0" }
vm_runtime = { git = "https://github.com/solana-labs/libra", tag = "v0.0.0.0" }
[dev-dependencies]
compiler = { git = "https://github.com/solana-labs/libra", tag = "v0.0.0.0" }
[lib]
crate-type = ["lib"]
name = "solana_move_loader_api"

View File

@ -0,0 +1,163 @@
// TODO
#![allow(dead_code)]
use failure::prelude::*;
use log::*;
use state_view::StateView;
use std::collections::HashMap;
use types::{
access_path::AccessPath,
account_address::AccountAddress,
account_config,
language_storage::ModuleId,
write_set::{WriteOp, WriteSet, WriteSetMut},
};
use vm::{errors::VMInvariantViolation, CompiledModule};
use vm_runtime::{
data_cache::RemoteCache,
identifier::create_access_path,
loaded_data::{struct_def::StructDef, types::Type},
value::Value,
};
/// An in-memory implementation of [`StateView`] and [`RemoteCache`] for the VM.
#[derive(Debug, Default)]
pub struct DataStore {
data: HashMap<AccessPath, Vec<u8>>,
}
impl DataStore {
/// Creates a new `DataStore` with the provided initial data.
pub fn new(data: HashMap<AccessPath, Vec<u8>>) -> Self {
DataStore { data }
}
/// Applies a [`WriteSet`] to this data store.
pub fn apply_write_set(&mut self, write_set: &WriteSet) {
for (access_path, write_op) in write_set {
match write_op {
WriteOp::Value(value) => {
self.set(access_path.clone(), value.clone());
}
WriteOp::Deletion => {
self.remove(access_path);
}
}
}
}
/// Returns a `WriteSet` for each account in the `DataStore`
pub fn into_write_sets(mut self) -> HashMap<AccountAddress, WriteSet> {
let mut write_set_muts: HashMap<AccountAddress, WriteSetMut> = HashMap::new();
for (access_path, value) in self.data.drain() {
match write_set_muts.get_mut(&access_path.address) {
Some(write_set_mut) => write_set_mut.push((access_path, WriteOp::Value(value))),
None => {
write_set_muts.insert(
access_path.address,
WriteSetMut::new(vec![(access_path, WriteOp::Value(value))]),
);
}
}
}
// Freeze each WriteSet
let mut write_sets: HashMap<AccountAddress, WriteSet> = HashMap::new();
for (address, write_set_mut) in write_set_muts.drain() {
write_sets.insert(address, write_set_mut.freeze().unwrap());
}
write_sets
}
/// Read an account's resource
pub fn read_account_resource(&self, addr: &AccountAddress) -> Option<Value> {
let access_path = create_access_path(&addr, account_config::account_struct_tag());
match self.data.get(&access_path) {
None => None,
Some(blob) => {
let account_type = get_account_struct_def();
match Value::simple_deserialize(blob, account_type) {
Ok(account) => Some(account),
Err(_) => None,
}
}
}
}
/// Sets a (key, value) pair within this data store.
///
/// Returns the previous data if the key was occupied.
pub fn set(&mut self, access_path: AccessPath, data_blob: Vec<u8>) -> Option<Vec<u8>> {
self.data.insert(access_path, data_blob)
}
/// Deletes a key from this data store.
///
/// Returns the previous data if the key was occupied.
pub fn remove(&mut self, access_path: &AccessPath) -> Option<Vec<u8>> {
self.data.remove(access_path)
}
/// Adds a [`CompiledModule`] to this data store.
///
/// Does not do any sort of verification on the module.
pub fn add_module(&mut self, module_id: &ModuleId, module: &CompiledModule) {
let access_path = AccessPath::from(module_id);
let mut value = vec![];
module
.serialize(&mut value)
.expect("serializing this module should work");
self.set(access_path, value);
}
/// Dumps the data store to stdout
pub fn dump(&self) {
for (access_path, value) in &self.data {
trace!("{:?}: \"{:?}\"", access_path, value.len());
}
}
}
impl StateView for DataStore {
fn get(&self, access_path: &AccessPath) -> Result<Option<Vec<u8>>> {
// Since the data is in-memory, it can't fail.
match self.data.get(access_path) {
None => Ok(None),
Some(value) => Ok(Some(value.clone())),
}
}
fn multi_get(&self, _access_paths: &[AccessPath]) -> Result<Vec<Option<Vec<u8>>>> {
unimplemented!();
}
fn is_genesis(&self) -> bool {
false
}
}
impl RemoteCache for DataStore {
fn get(
&self,
access_path: &AccessPath,
) -> ::std::result::Result<Option<Vec<u8>>, VMInvariantViolation> {
Ok(StateView::get(self, access_path).expect("it should not error"))
}
}
// TODO: internal Libra function and very likely to break soon, need something better
fn get_account_struct_def() -> StructDef {
// STRUCT DEF StructDef(StructDefInner { field_definitions: [ByteArray,
// Struct(StructDef(StructDefInner { field_definitions: [U64] })), U64, U64,
// U64] }) let coin = StructDef(StructDefInner { field_definitions:
// [Type::U64] })
let int_type = Type::U64;
let byte_array_type = Type::ByteArray;
let coin = Type::Struct(StructDef::new(vec![int_type.clone()]));
StructDef::new(vec![
byte_array_type,
coin,
int_type.clone(),
int_type.clone(),
int_type.clone(),
])
}

Binary file not shown.

View File

@ -0,0 +1,399 @@
const MOVE_LOADER_PROGRAM_ID: [u8; 32] = [
5, 91, 237, 31, 90, 253, 197, 145, 157, 236, 147, 43, 6, 5, 157, 238, 63, 151, 181, 165, 118,
224, 198, 97, 103, 136, 113, 64, 0, 0, 0, 0,
];
solana_sdk::solana_name_id!(
MOVE_LOADER_PROGRAM_ID,
"MvLdr11111111111111111111111111111111111111"
);
mod data_store;
use bytecode_verifier::{VerifiedModule, VerifiedScript};
use data_store::DataStore;
use log::*;
use serde_derive::{Deserialize, Serialize};
use solana_sdk::{
account::KeyedAccount, instruction::InstructionError, loader_instruction::LoaderInstruction,
pubkey::Pubkey,
};
use std::convert::TryInto;
use types::{
account_address::AccountAddress,
transaction::{Program, TransactionArgument, TransactionOutput, TransactionStatus},
write_set::WriteSet,
};
use vm::{
access::ModuleAccess, file_format::CompiledScript, transaction_metadata::TransactionMetadata,
};
use vm_cache_map::Arena;
use vm_runtime::{
code_cache::{
module_adapter::ModuleFetcherImpl,
module_cache::{BlockModuleCache, ModuleCache, VMModuleCache},
},
static_verify_program,
txn_executor::TransactionExecutor,
value::Local,
};
/// Type of exchange account, account's user data is populated with this enum
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub enum LibraAccountState {
/// No data for this account yet
Unallocated,
/// Write set containing a Libra account's data
WriteSet(WriteSet),
}
fn arguments_to_locals(args: Vec<TransactionArgument>) -> Vec<Local> {
let mut locals = vec![];
for arg in args.into_iter() {
locals.push(match arg {
TransactionArgument::U64(i) => Local::u64(i),
TransactionArgument::Address(a) => Local::address(a),
TransactionArgument::ByteArray(b) => Local::bytearray(b),
TransactionArgument::String(s) => Local::string(s),
});
}
locals
}
fn to_array_32(array: &[u8]) -> &[u8; 32] {
array.try_into().expect("slice with incorrect length")
}
pub fn execute(
script: VerifiedScript,
args: Vec<TransactionArgument>,
modules: Vec<VerifiedModule>,
data_store: &DataStore,
) -> TransactionOutput {
let allocator = Arena::new();
let code_cache = VMModuleCache::new(&allocator);
let module_cache = BlockModuleCache::new(&code_cache, ModuleFetcherImpl::new(data_store));
for m in modules {
module_cache.cache_module(m);
}
let main_module = script.into_module();
let module_id = main_module.self_id();
module_cache.cache_module(main_module);
let txn_metadata = TransactionMetadata::default();
let mut vm = TransactionExecutor::new(&module_cache, data_store, txn_metadata);
let result = vm.execute_function(&module_id, &"main".to_string(), arguments_to_locals(args));
vm.make_write_set(vec![], result).unwrap()
}
pub fn process_instruction(
_program_id: &Pubkey,
keyed_accounts: &mut [KeyedAccount],
ix_data: &[u8],
) -> Result<(), InstructionError> {
solana_logger::setup();
if let Ok(instruction) = bincode::deserialize(ix_data) {
match instruction {
LoaderInstruction::Write { offset, bytes } => {
const PROGRAM_INDEX: usize = 0;
if keyed_accounts[PROGRAM_INDEX].signer_key().is_none() {
warn!("key[0] did not sign the transaction");
return Err(InstructionError::GenericError);
}
let offset = offset as usize;
let len = bytes.len();
debug!("Write: offset={} length={}", offset, len);
if keyed_accounts[PROGRAM_INDEX].account.data.len() < offset + len {
warn!(
"Write overflow: {} < {}",
keyed_accounts[PROGRAM_INDEX].account.data.len(),
offset + len
);
return Err(InstructionError::GenericError);
}
keyed_accounts[PROGRAM_INDEX].account.data[offset..offset + len]
.copy_from_slice(&bytes);
}
LoaderInstruction::Finalize => {
const PROGRAM_INDEX: usize = 0;
if keyed_accounts[PROGRAM_INDEX].signer_key().is_none() {
warn!("key[0] did not sign the transaction");
return Err(InstructionError::GenericError);
}
keyed_accounts[PROGRAM_INDEX].account.executable = true;
info!(
"Finalize: account {:?}",
keyed_accounts[PROGRAM_INDEX].signer_key().unwrap()
);
}
LoaderInstruction::InvokeMain { data } => {
const PROGRAM_INDEX: usize = 0;
const GENESIS_INDEX: usize = 1;
if keyed_accounts.len() < 2 {
error!("Need at least program and genesis accounts");
return Err(InstructionError::InvalidArgument);
}
if keyed_accounts[PROGRAM_INDEX].account.owner
!= Pubkey::new(&MOVE_LOADER_PROGRAM_ID)
{
error!("Move program account not owned by Move loader");
return Err(InstructionError::InvalidArgument);
}
if !keyed_accounts[PROGRAM_INDEX].account.executable {
error!("Move program account not executable");
return Err(InstructionError::InvalidArgument);
}
// TODO: Return errors instead of panicking
let args: Vec<TransactionArgument> = bincode::deserialize(&data).unwrap();
// Dump Libra account data into data store
let mut data_store = DataStore::default();
for keyed_account in keyed_accounts[GENESIS_INDEX..].iter() {
if let LibraAccountState::WriteSet(write_set) =
bincode::deserialize(&keyed_account.account.data).unwrap()
{
data_store.apply_write_set(&write_set);
}
}
let program: Program =
serde_json::from_slice(&keyed_accounts[0].account.data).unwrap();
let compiled_script = CompiledScript::deserialize(program.code()).unwrap();
let modules = vec![];
let sender_address = AccountAddress::default();
// TODO: This function calls `.expect()` internally, need an error friendly version
let (verified_script, modules) =
static_verify_program(&sender_address, compiled_script, modules)
.expect("verification failure");
let output = execute(verified_script, args, modules, &data_store);
for event in output.events() {
debug!("Event: {:?}", event);
}
if let TransactionStatus::Discard(status) = output.status() {
error!("Execution failed: {:?}", status);
return Err(InstructionError::GenericError);
}
data_store.apply_write_set(&output.write_set());
// Dump Libra account data back into Solana account data
let mut write_sets = data_store.into_write_sets();
for (i, keyed_account) in keyed_accounts[GENESIS_INDEX..].iter_mut().enumerate() {
let address = if i == 0 {
// TODO: Remove this special case for genesis when genesis contains real pubkey
AccountAddress::default()
} else {
// let mut address = [0_u8; 32];
// address.copy_from_slice(keyed_account.unsigned_key().as_ref());
AccountAddress::new(*to_array_32(keyed_account.unsigned_key().as_ref()))
};
let write_set = write_sets.remove(&address).unwrap();
keyed_account.account.data.clear();
let writer = std::io::BufWriter::new(&mut keyed_account.account.data);
bincode::serialize_into(writer, &LibraAccountState::WriteSet(write_set))
.unwrap();
}
if !write_sets.is_empty() {
error!("Missing keyed accounts");
return Err(InstructionError::GenericError);
}
}
}
} else {
warn!("Invalid program transaction: {:?}", ix_data);
return Err(InstructionError::GenericError);
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use compiler::Compiler;
use language_e2e_tests::account::AccountResource;
use lazy_static::lazy_static;
use proto_conv::FromProto;
use protobuf::parse_from_bytes;
use solana_sdk::account::Account;
use std::{fs::File, io::prelude::*, path::PathBuf};
use types::transaction::{SignedTransaction, TransactionPayload};
// TODO: Cleanup copypasta.
#[test]
fn test_invoke_main() {
solana_logger::setup();
let program_id = Pubkey::new(&MOVE_LOADER_PROGRAM_ID);
let code = "main() { return; }";
let address = AccountAddress::default();
let compiler = Compiler {
code,
address,
..Compiler::default()
};
let compiled_program = compiler.into_compiled_program().expect("Failed to compile");
let mut script = vec![];
compiled_program
.script
.serialize(&mut script)
.expect("Unable to serialize script");
let mut modules = vec![];
for m in compiled_program.modules.iter() {
let mut buf = vec![];
m.serialize(&mut buf).expect("Unable to serialize module");
modules.push(buf);
}
let move_program_pubkey = Pubkey::new_rand();
let program = Program::new(script, modules, vec![]);
let program_bytes = serde_json::to_vec(&program).unwrap();
let mut move_program_account = Account {
lamports: 1,
data: program_bytes,
owner: program_id,
executable: true,
};
let (genesis_pubkey, mut genesis_account) = get_genesis();
let mut keyed_accounts = vec![
KeyedAccount::new(&move_program_pubkey, false, &mut move_program_account),
KeyedAccount::new(&genesis_pubkey, false, &mut genesis_account),
];
let args: Vec<TransactionArgument> = vec![];
let data = bincode::serialize(&args).unwrap();
let ix = LoaderInstruction::InvokeMain { data };
let ix_data = bincode::serialize(&ix).unwrap();
// Ensure no panic
process_instruction(&program_id, &mut keyed_accounts, &ix_data).unwrap();
}
#[test]
fn test_mint() {
solana_logger::setup();
let program_id = Pubkey::new(&MOVE_LOADER_PROGRAM_ID);
let code = "
import 0x0.LibraAccount;
import 0x0.LibraCoin;
main(payee: address, amount: u64) {
LibraAccount.mint_to_address(move(payee), move(amount));
return;
}
";
let address = AccountAddress::default();
let compiler = Compiler {
code,
address,
..Compiler::default()
};
let compiled_program = compiler.into_compiled_program().expect("Failed to compile");
let mut script = vec![];
compiled_program
.script
.serialize(&mut script)
.expect("Unable to serialize script");
let mut modules = vec![];
for m in compiled_program.modules.iter() {
let mut buf = vec![];
m.serialize(&mut buf).expect("Unable to serialize module");
modules.push(buf);
}
let move_program_pubkey = Pubkey::new_rand();
let program = Program::new(script, modules, vec![]);
let program_bytes = serde_json::to_vec(&program).unwrap();
let mut move_program_account = Account {
lamports: 1,
data: program_bytes,
owner: program_id,
executable: true,
};
let (genesis_pubkey, mut genesis_account) = get_genesis();
let receiver_pubkey = Pubkey::new_rand();
let receiver_bytes = bincode::serialize(&LibraAccountState::Unallocated).unwrap();
let mut receiver_account = Account {
lamports: 1,
data: receiver_bytes,
owner: program_id,
executable: true,
};
let mut keyed_accounts = vec![
KeyedAccount::new(&move_program_pubkey, false, &mut move_program_account),
KeyedAccount::new(&genesis_pubkey, false, &mut genesis_account),
KeyedAccount::new(&receiver_pubkey, false, &mut receiver_account),
];
let receiver_addr = AccountAddress::new(to_array_32(receiver_pubkey.as_ref()).clone());
let mint_amount = 42;
let mut args: Vec<TransactionArgument> = Vec::new();
args.push(TransactionArgument::Address(receiver_addr));
args.push(TransactionArgument::U64(mint_amount));
let data = bincode::serialize(&args).unwrap();
let ix = LoaderInstruction::InvokeMain { data };
let ix_data = bincode::serialize(&ix).unwrap();
process_instruction(&program_id, &mut keyed_accounts, &ix_data).unwrap();
let mut data_store = DataStore::default();
match bincode::deserialize(&keyed_accounts[2].account.data).unwrap() {
LibraAccountState::Unallocated => panic!("Invalid account state"),
LibraAccountState::WriteSet(write_set) => data_store.apply_write_set(&write_set),
}
let updated_receiver = data_store.read_account_resource(&receiver_addr).unwrap();
assert_eq!(42, AccountResource::read_balance(&updated_receiver));
assert_eq!(0, AccountResource::read_sequence_number(&updated_receiver));
}
// TODO: Need a mechanism to initially encode the genesis and enforce it
lazy_static! {
/// The write set encoded in the genesis transaction.
static ref GENESIS_WRITE_SET: WriteSet = {
let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
path.pop();
path.push("move_loader_api/src/genesis.blob");
let mut f = File::open(&path).unwrap();
let mut bytes = vec![];
f.read_to_end(&mut bytes).unwrap();
let txn = SignedTransaction::from_proto(parse_from_bytes(&bytes).unwrap()).unwrap();
match txn.payload() {
TransactionPayload::WriteSet(ws) => ws.clone(),
_ => panic!("Expected writeset txn in genesis txn"),
}
};
}
fn get_genesis() -> (Pubkey, Account) {
let genesis_pubkey = Pubkey::new_rand();
let write_set = GENESIS_WRITE_SET.clone();
let genesis_bytes = bincode::serialize(&LibraAccountState::WriteSet(write_set))
.expect("Failed to serialize genesis WriteSet");
let genesis_account = Account {
lamports: 1,
data: genesis_bytes,
owner: Pubkey::new(&MOVE_LOADER_PROGRAM_ID),
executable: false,
};
(genesis_pubkey, genesis_account)
}
}

View File

@ -0,0 +1,2 @@
/farf/
/target/

3536
programs/move_loader_program/Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,19 @@
[package]
name = "solana-move-loader-program"
version = "0.17.0"
description = "Solana Move Loader"
authors = ["Solana Maintainers <maintainers@solana.com>"]
repository = "https://github.com/solana-labs/solana"
license = "Apache-2.0"
homepage = "https://solana.com/"
edition = "2018"
[dependencies]
log = "0.4.2"
solana-logger = { path = "../../logger", version = "0.17.0" }
solana-sdk = { path = "../../sdk", version = "0.17.0" }
solana-move-loader-api = { path = "../move_loader_api", version = "0.17.0" }
[lib]
crate-type = ["lib", "cdylib"]
name = "solana_move_loader_program"

View File

@ -0,0 +1 @@
nightly-2019-07-08

View File

@ -0,0 +1,12 @@
#[macro_export]
macro_rules! solana_move_loader_program {
() => {
(
"solana_move_loader_program".to_string(),
solana_move_loader_api::id(),
)
};
}
use solana_move_loader_api::process_instruction;
solana_sdk::solana_entrypoint!(process_instruction);