From a7b092a6111b53055e148b98bcefe6ea82aa8422 Mon Sep 17 00:00:00 2001 From: Armani Ferrante Date: Tue, 9 Mar 2021 05:40:09 -0800 Subject: [PATCH] docker, cli, lang: Deterministic and verifiable builds (#100) --- CHANGELOG.md | 7 +- cli/src/main.rs | 240 +++++++++++++++++++++++++++++++++++++--- docker/Makefile | 19 ++++ docker/build/Dockerfile | 38 +++++++ lang/syn/src/idl.rs | 28 ++--- 5 files changed, 300 insertions(+), 32 deletions(-) create mode 100644 docker/Makefile create mode 100644 docker/build/Dockerfile diff --git a/CHANGELOG.md b/CHANGELOG.md index f0949f48b..dc093967e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,10 +14,11 @@ incremented for features. ## Features * ts: Allow preloading instructions for state rpc transactions ([cf9c84](https://github.com/project-serum/anchor/commit/cf9c847e4144989b5bc1936149d171e90204777b)). -* ts: Export sighash coder function. -* cli: Specify programs to embed into local validator genesis via Anchor.toml while testing. -* cli: Allow skipping the creation of a local validator when testing against localnet. +* ts: Export sighash coder function ([734c75](https://github.com/project-serum/anchor/commit/734c751882f43beec7ea3f0f4d988b502e3f24e4)). +* cli: Specify programs to embed into local validator genesis via Anchor.toml while testing ([b3803a](https://github.com/project-serum/anchor/commit/b3803aec03fbbae1a794c9aa6a789e6cb58fda99)). +* cli: Allow skipping the creation of a local validator when testing against localnet ([#93](https://github.com/project-serum/anchor/pull/93)). * cli: Adds support for tests with Typescript ([#94](https://github.com/project-serum/anchor/pull/94)). +* cli: Deterministic and verifiable builds ([#100](https://github.com/project-serum/anchor/pull/100)). ## Fixes diff --git a/cli/src/main.rs b/cli/src/main.rs index f13202901..26d75e95e 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -1,3 +1,5 @@ +//! CLI for workspace management of anchor programs. + use crate::config::{read_all_programs, Config, Program}; use anchor_lang::idl::IdlAccount; use anchor_lang::{AccountDeserialize, AnchorDeserialize, AnchorSerialize}; @@ -12,6 +14,8 @@ use serde::{Deserialize, Serialize}; use solana_client::rpc_client::RpcClient; use solana_client::rpc_config::RpcSendTransactionConfig; use solana_program::instruction::{AccountMeta, Instruction}; +use solana_sdk::account_utils::StateMut; +use solana_sdk::bpf_loader_upgradeable::UpgradeableLoaderState; use solana_sdk::commitment_config::CommitmentConfig; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::Keypair; @@ -45,6 +49,16 @@ pub enum Command { /// Output directory for the IDL. #[clap(short, long)] idl: Option, + /// True if the build artifiact needs to be deterministic and verifiable. + #[clap(short, long)] + verifiable: bool, + }, + /// Verifies the on-chain bytecode matches the locally compiled artifact. + /// Run this command inside a program subdirectory, i.e., in the dir + /// containing the program's Cargo.toml. + Verify { + /// The deployed program to compare against. + program_id: Pubkey, }, /// Runs integration tests against a localnetwork. Test { @@ -82,6 +96,10 @@ pub enum Command { url: Option, #[clap(short, long)] keypair: Option, + /// True if the build should be verifiable. If deploying to mainnet, + /// this should almost always be set. + #[clap(short, long)] + verifiable: bool, }, /// Upgrades a single program. The configured wallet must be the upgrade /// authority. @@ -157,7 +175,8 @@ fn main() -> Result<()> { match opts.command { Command::Init { name, typescript } => init(name, typescript), Command::New { name } => new(name), - Command::Build { idl } => build(idl), + Command::Build { idl, verifiable } => build(idl, verifiable), + Command::Verify { program_id } => verify(program_id), Command::Deploy { url, keypair } => deploy(url, keypair), Command::Upgrade { program_id, @@ -165,7 +184,11 @@ fn main() -> Result<()> { } => upgrade(program_id, program_filepath), Command::Idl { subcmd } => idl(subcmd), Command::Migrate { url } => migrate(url), - Command::Launch { url, keypair } => launch(url, keypair), + Command::Launch { + url, + keypair, + verifiable, + } => launch(url, keypair, verifiable), Command::Test { skip_deploy, skip_local_validator, @@ -257,7 +280,7 @@ fn new_program(name: &str) -> Result<()> { Ok(()) } -fn build(idl: Option) -> Result<()> { +fn build(idl: Option, verifiable: bool) -> Result<()> { let (cfg, path, cargo) = Config::discover()?.expect("Not in workspace."); let idl_out = match idl { Some(idl) => Some(PathBuf::from(idl)), @@ -271,8 +294,8 @@ fn build(idl: Option) -> Result<()> { } }; match cargo { - None => build_all(&cfg, path, idl_out)?, - Some(ct) => build_cwd(ct, idl_out)?, + None => build_all(&cfg, path, idl_out, verifiable)?, + Some(ct) => build_cwd(path.as_path(), ct, idl_out, verifiable)?, }; set_workspace_dir_or_exit(); @@ -280,27 +303,129 @@ fn build(idl: Option) -> Result<()> { Ok(()) } -fn build_all(_cfg: &Config, cfg_path: PathBuf, idl_out: Option) -> Result<()> { - match cfg_path.parent() { +fn build_all( + _cfg: &Config, + cfg_path: PathBuf, + idl_out: Option, + verifiable: bool, +) -> Result<()> { + let cur_dir = std::env::current_dir()?; + let r = match cfg_path.parent() { None => Err(anyhow!("Invalid Anchor.toml at {}", cfg_path.display())), Some(parent) => { let files = fs::read_dir(parent.join("programs"))?; for f in files { let p = f?.path(); - build_cwd(p.join("Cargo.toml"), idl_out.clone())?; + build_cwd( + cfg_path.as_path(), + p.join("Cargo.toml"), + idl_out.clone(), + verifiable, + )?; } Ok(()) } - } + }; + std::env::set_current_dir(cur_dir)?; + r } // Runs the build command outside of a workspace. -fn build_cwd(cargo_toml: PathBuf, idl_out: Option) -> Result<()> { +fn build_cwd( + cfg_path: &Path, + cargo_toml: PathBuf, + idl_out: Option, + verifiable: bool, +) -> Result<()> { match cargo_toml.parent() { None => return Err(anyhow!("Unable to find parent")), Some(p) => std::env::set_current_dir(&p)?, }; + match verifiable { + false => _build_cwd(idl_out), + true => build_cwd_verifiable(cfg_path.parent().unwrap()), + } +} +// Builds an anchor program in a docker image and copies the build artifacts +// into the `target/` directory. +fn build_cwd_verifiable(workspace_dir: &Path) -> Result<()> { + // Docker vars. + let container_name = "anchor-program"; + let image_name = "projectserum/build"; + let volume_mount = format!( + "{}:/workdir", + workspace_dir.canonicalize()?.display().to_string() + ); + + // Create output dirs. + fs::create_dir_all(workspace_dir.join("target/deploy"))?; + fs::create_dir_all(workspace_dir.join("target/idl"))?; + + // Build the program in docker. + let exit = std::process::Command::new("docker") + .args(&[ + "run", + "--name", + &container_name, + "-v", + &volume_mount, + &image_name, + "anchor", + "build", + ]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .output() + .map_err(|e| anyhow::format_err!("{}", e.to_string()))?; + if !exit.status.success() { + println!("Error building program"); + return Ok(()); + } + + let idl = extract_idl("src/lib.rs")?; + + // Copy the binary out of the docker image. + let out_file = format!("../../target/deploy/{}.so", idl.name); + let bin_artifact = format!("{}:/workdir/target/deploy/{}.so", container_name, idl.name); + let exit = std::process::Command::new("docker") + .args(&["cp", &bin_artifact, &out_file]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .output() + .map_err(|e| anyhow::format_err!("{}", e.to_string()))?; + if !exit.status.success() { + return Ok(()); + } + + // Copy the idl out of the docker image. + let out_file = format!("../../target/idl/{}.json", idl.name); + let idl_artifact = format!("{}:/workdir/target/idl/{}.json", container_name, idl.name); + let exit = std::process::Command::new("docker") + .args(&["cp", &idl_artifact, &out_file]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .output() + .map_err(|e| anyhow::format_err!("{}", e.to_string()))?; + if !exit.status.success() { + return Ok(()); + } + + // Remove the docker image. + let exit = std::process::Command::new("docker") + .args(&["rm", &container_name]) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .output() + .map_err(|e| anyhow::format_err!("{}", e.to_string()))?; + if !exit.status.success() { + std::process::exit(exit.status.code().unwrap_or(1)); + } + + Ok(()) +} + +fn _build_cwd(idl_out: Option) -> Result<()> { let exit = std::process::Command::new("cargo") .arg("build-bpf") .stdout(Stdio::inherit()) @@ -322,6 +447,83 @@ fn build_cwd(cargo_toml: PathBuf, idl_out: Option) -> Result<()> { write_idl(&idl, OutFile::File(out)) } +fn verify(program_id: Pubkey) -> Result<()> { + let (cfg, _path, cargo) = Config::discover()?.expect("Not in workspace."); + let cargo = cargo.ok_or(anyhow!("Must be inside program subdirectory."))?; + let program_dir = cargo.parent().unwrap(); + + // Build the program we want to verify. + let cur_dir = std::env::current_dir()?; + build(None, true)?; + std::env::set_current_dir(&cur_dir)?; + + // Verify IDL. + std::env::set_current_dir(program_dir)?; + let local_idl = extract_idl("src/lib.rs")?; + let deployed_idl = fetch_idl(program_id)?; + if local_idl != deployed_idl { + println!("Error: IDLs don't match"); + std::process::exit(1); + } + + // Verify binary. + let bin_path = program_dir + .join("../../target/deploy/") + .join(format!("{}.so", local_idl.name)); + verify_bin(program_id, &bin_path, cfg.cluster.url())?; + + println!("{} is verified.", program_id); + + Ok(()) +} + +fn verify_bin(program_id: Pubkey, bin_path: &Path, cluster: &str) -> Result<()> { + let client = RpcClient::new(cluster.to_string()); + + // Get the deployed build artifacts. + let deployed_bin = { + let account = client + .get_account_with_commitment(&program_id, CommitmentConfig::default())? + .value + .map_or(Err(anyhow!("Account not found")), Ok)?; + match account.state()? { + UpgradeableLoaderState::Program { + programdata_address, + } => client + .get_account_with_commitment(&programdata_address, CommitmentConfig::default())? + .value + .map_or(Err(anyhow!("Account not found")), Ok)? + .data[UpgradeableLoaderState::programdata_data_offset().unwrap_or(0)..] + .to_vec(), + UpgradeableLoaderState::Buffer { .. } => { + let offset = UpgradeableLoaderState::buffer_data_offset().unwrap_or(0); + account.data[offset..].to_vec() + } + _ => return Err(anyhow!("Invalid program id")), + } + }; + let mut local_bin = { + let mut f = File::open(bin_path)?; + let mut contents = vec![]; + f.read_to_end(&mut contents)?; + contents + }; + + // The deployed program probably has zero bytes appended. The default is + // 2x the binary size in case of an upgrade. + if local_bin.len() < deployed_bin.len() { + local_bin.append(&mut vec![0; deployed_bin.len() - local_bin.len()]); + } + + // Finally, check the bytes. + if local_bin != deployed_bin { + println!("Error: Binaries don't match"); + std::process::exit(1); + } + + Ok(()) +} + // Fetches an IDL for the given program_id. fn fetch_idl(program_id: Pubkey) -> Result { let cfg = Config::discover()?.expect("Inside a workspace").0; @@ -517,6 +719,10 @@ fn idl_clear(cfg: &Config, program_id: &Pubkey) -> Result<()> { // and sending multiple transactions in the event the IDL doesn't fit into // a single transaction. fn idl_write(cfg: &Config, program_id: &Pubkey, idl: &Idl) -> Result<()> { + // Remove the metadata before deploy. + let mut idl = idl.clone(); + idl.metadata = None; + // Misc. let idl_address = IdlAccount::address(program_id); let keypair = solana_sdk::signature::read_keypair_file(&cfg.wallet.to_string()) @@ -525,7 +731,7 @@ fn idl_write(cfg: &Config, program_id: &Pubkey, idl: &Idl) -> Result<()> { // Serialize and compress the idl. let idl_data = { - let json_bytes = serde_json::to_vec(idl)?; + let json_bytes = serde_json::to_vec(&idl)?; let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); e.write_all(&json_bytes)?; e.finish()? @@ -612,7 +818,7 @@ fn test(skip_deploy: bool, skip_local_validator: bool) -> Result<()> { // Bootup validator, if needed. let validator_handle = match cfg.cluster.url() { "http://127.0.0.1:8899" => { - build(None)?; + build(None, false)?; let flags = match skip_deploy { true => None, false => Some(genesis_flags(cfg)?), @@ -624,6 +830,7 @@ fn test(skip_deploy: bool, skip_local_validator: bool) -> Result<()> { } _ => { if !skip_deploy { + build(None, false)?; deploy(None, None)?; } None @@ -797,8 +1004,6 @@ fn deploy(url: Option, keypair: Option) -> Result<()> { fn _deploy(url: Option, keypair: Option) -> Result> { with_workspace(|cfg, _path, _cargo| { - build(None)?; - // Fallback to config vars if not provided via CLI. let url = url.unwrap_or_else(|| cfg.cluster.url().to_string()); let keypair = keypair.unwrap_or_else(|| cfg.wallet.to_string()); @@ -887,8 +1092,9 @@ fn upgrade(program_id: Pubkey, program_filepath: String) -> Result<()> { }) } -fn launch(url: Option, keypair: Option) -> Result<()> { +fn launch(url: Option, keypair: Option, verifiable: bool) -> Result<()> { // Build and deploy. + build(None, verifiable)?; let programs = _deploy(url.clone(), keypair.clone())?; with_workspace(|cfg, _path, _cargo| { @@ -1020,6 +1226,10 @@ fn migrate(url: Option) -> Result<()> { let cur_dir = std::env::current_dir()?; let module_path = format!("{}/migrations/deploy.js", cur_dir.display()); let deploy_script_host_str = template::deploy_script_host(&url, &module_path); + + if !Path::new(".anchor").exists() { + fs::create_dir(".anchor")?; + } std::env::set_current_dir(".anchor")?; std::fs::write("deploy.js", deploy_script_host_str)?; diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 000000000..71e185e17 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,19 @@ +IMG_ORG ?= projectserum +IMG_VER ?= latest + +WORKDIR=$(PWD) + +.PHONY: build build-push build-shell + +default: + +build: build/Dockerfile + @docker build $@ -t $(IMG_ORG)/$@:$(IMG_VER) + +build-push: + @docker push $(IMG_ORG)/anchorbuild:$(IMG_VER) + +build-shell: + @docker run -ti --rm --net=host \ + -v $(WORKDIR)/..:/workdir \ + $(IMG_ORG)/build:$(IMG_VER) bash diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile new file mode 100644 index 000000000..b2fc5c6e0 --- /dev/null +++ b/docker/build/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:18.04 + +ARG DEBIAN_FRONTEND=noninteractive + +ARG SOLANA_CHANNEL=v1.2.17 +ARG SOLANA_CLI=v1.5.6 + +ENV HOME="/root" +ENV PATH="${HOME}/.cargo/bin:${PATH}" +ENV PATH="${HOME}/.local/share/solana/install/active_release/bin:${PATH}" + +# Install base utilities. +RUN mkdir -p /workdir && mkdir -p /tmp && \ + apt-get update -qq && apt-get upgrade -qq && apt-get install -qq \ + build-essential git curl wget jq pkg-config python3-pip \ + libssl-dev libudev-dev + +# Install rust. +RUN curl "https://sh.rustup.rs" -sfo rustup.sh && \ + sh rustup.sh -y && \ + rustup component add rustfmt clippy + +# Install Solana tools. +RUN curl -sSf https://raw.githubusercontent.com/solana-labs/solana/${SOLANA_CLI}/install/solana-install-init.sh | sh -s - ${SOLANA_CLI} && \ + # BPF sdk. + curl -L --retry 5 --retry-delay 2 -o bpf-sdk.tar.bz2 http://solana-sdk.s3.amazonaws.com/${SOLANA_CHANNEL}/bpf-sdk.tar.bz2 && \ + rm -rf bpf-sdk && \ + mkdir -p bpf-sdk && \ + tar jxf bpf-sdk.tar.bz2 && \ + rm -f bpf-sdk.tar.bz2 + +# Install anchor. +RUN cargo install --git https://github.com/project-serum/anchor anchor-cli --locked + +# Build a dummy program to bootstrap the BPF SDK (doing this speeds up builds). +RUN mkdir -p /tmp && cd tmp && anchor init dummy && cd dummy && anchor build + +WORKDIR /workdir diff --git a/lang/syn/src/idl.rs b/lang/syn/src/idl.rs index 760092d62..2c9438f40 100644 --- a/lang/syn/src/idl.rs +++ b/lang/syn/src/idl.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct Idl { pub version: String, pub name: String, @@ -17,7 +17,7 @@ pub struct Idl { pub metadata: Option, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct IdlState { #[serde(rename = "struct")] pub strct: IdlTypeDef, @@ -26,7 +26,7 @@ pub struct IdlState { pub type IdlStateMethod = IdlIx; -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct IdlIx { pub name: String, pub accounts: Vec, @@ -34,14 +34,14 @@ pub struct IdlIx { } // A single struct deriving `Accounts`. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct IdlAccounts { pub name: String, pub accounts: Vec, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(untagged)] pub enum IdlAccountItem { IdlAccount(IdlAccount), @@ -49,7 +49,7 @@ pub enum IdlAccountItem { } // A single field in the accounts struct. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct IdlAccount { pub name: String, @@ -57,42 +57,42 @@ pub struct IdlAccount { pub is_signer: bool, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct IdlField { pub name: String, #[serde(rename = "type")] pub ty: IdlType, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct IdlTypeDef { pub name: String, #[serde(rename = "type")] pub ty: IdlTypeDefTy, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "lowercase", tag = "kind")] pub enum IdlTypeDefTy { Struct { fields: Vec }, Enum { variants: Vec }, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct EnumVariant { pub name: String, #[serde(skip_serializing_if = "Option::is_none", default)] pub fields: Option, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(untagged)] pub enum EnumFields { Named(Vec), Tuple(Vec), } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] pub enum IdlType { Bool, @@ -114,7 +114,7 @@ pub enum IdlType { Vec(Box), } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct IdlTypePublicKey; impl std::str::FromStr for IdlType { @@ -166,7 +166,7 @@ impl std::str::FromStr for IdlType { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct IdlErrorCode { pub code: u32, pub name: String,