Bump bpf-tools to v1.2 and get rid of xargo

2021-03-16 16:32:42 +01:00 · 2021-03-16 16:32:42 +01:00 · 14e45155e4
parent 3d8c4f8389
commit 14e45155e4
43 changed files with 187 additions and 193 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -4118,9 +4118,12 @@ dependencies = [
 name = "solana-cargo-build-bpf"
 version = "1.7.0"
 dependencies = [
+ "bzip2",
 "cargo_metadata",
 "clap",
+ "solana-download-utils",
 "solana-sdk",
+ "tar",
 ]

 [[package]]
--- a/ci/test-stable.sh
+++ b/ci/test-stable.sh
@ -25,9 +25,6 @@ source scripts/ulimit-n.sh
 test -d target/debug/bpf && find target/debug/bpf -name '*.d' -delete
 test -d target/release/bpf && find target/release/bpf -name '*.d' -delete

-# Clear the BPF sysroot files, they are not automatically rebuilt
-rm -rf target/xargo # Issue #3105
-
 # Limit compiler jobs to reduce memory usage
 # on machines with 2gb/thread of memory
 NPROC=$(nproc)
--- a/programs/bpf/rust/128bit/Xargo.toml
+++ b/programs/bpf/rust/128bit/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/128bit_dep/Xargo.toml
+++ b/programs/bpf/rust/128bit_dep/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/alloc/Xargo.toml
+++ b/programs/bpf/rust/alloc/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/call_depth/Xargo.toml
+++ b/programs/bpf/rust/call_depth/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/caller_access/Xargo.toml
+++ b/programs/bpf/rust/caller_access/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/custom_heap/Xargo.toml
+++ b/programs/bpf/rust/custom_heap/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/dep_crate/Xargo.toml
+++ b/programs/bpf/rust/dep_crate/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/deprecated_loader/Xargo.toml
+++ b/programs/bpf/rust/deprecated_loader/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/dup_accounts/Xargo.toml
+++ b/programs/bpf/rust/dup_accounts/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/error_handling/Xargo.toml
+++ b/programs/bpf/rust/error_handling/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/external_spend/Xargo.toml
+++ b/programs/bpf/rust/external_spend/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/finalize/Xargo.toml
+++ b/programs/bpf/rust/finalize/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/instruction_introspection/Xargo.toml
+++ b/programs/bpf/rust/instruction_introspection/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/invoke/Xargo.toml
+++ b/programs/bpf/rust/invoke/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/invoke_and_error/Xargo.toml
+++ b/programs/bpf/rust/invoke_and_error/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/invoke_and_ok/Xargo.toml
+++ b/programs/bpf/rust/invoke_and_ok/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/invoke_and_return/Xargo.toml
+++ b/programs/bpf/rust/invoke_and_return/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/invoked/Xargo.toml
+++ b/programs/bpf/rust/invoked/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/iter/Xargo.toml
+++ b/programs/bpf/rust/iter/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/many_args/Xargo.toml
+++ b/programs/bpf/rust/many_args/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/many_args_dep/Xargo.toml
+++ b/programs/bpf/rust/many_args_dep/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/mem/Xargo.toml
+++ b/programs/bpf/rust/mem/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/noop/Xargo.toml
+++ b/programs/bpf/rust/noop/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/panic/Xargo.toml
+++ b/programs/bpf/rust/panic/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/param_passing/Xargo.toml
+++ b/programs/bpf/rust/param_passing/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/param_passing_dep/Xargo.toml
+++ b/programs/bpf/rust/param_passing_dep/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/rand/Xargo.toml
+++ b/programs/bpf/rust/rand/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/ristretto/Xargo.toml
+++ b/programs/bpf/rust/ristretto/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/ro_modify/Xargo.toml
+++ b/programs/bpf/rust/ro_modify/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/sanity/Xargo.toml
+++ b/programs/bpf/rust/sanity/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/sha256/Xargo.toml
+++ b/programs/bpf/rust/sha256/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/spoof1/Xargo.toml
+++ b/programs/bpf/rust/spoof1/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/spoof1_system/Xargo.toml
+++ b/programs/bpf/rust/spoof1_system/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/sysval/Xargo.toml
+++ b/programs/bpf/rust/sysval/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/upgradeable/Xargo.toml
+++ b/programs/bpf/rust/upgradeable/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/programs/bpf/rust/upgraded/Xargo.toml
+++ b/programs/bpf/rust/upgraded/Xargo.toml
@ -1,2 +0,0 @@
-[target.bpfel-unknown-unknown.dependencies.std]
-features = []
--- a/sdk/bpf/env.sh
+++ b/sdk/bpf/env.sh
@ -27,13 +27,3 @@ export RUSTFLAGS="
    -C link-arg=--threads=1 \
    -C link-arg=--entry=entrypoint \
    -C linker=$bpf_sdk/dependencies/bpf-tools/llvm/bin/ld.lld"
-
-# CARGO may be set if run from within cargo, causing
-# incompatibilities between cargo and xargo versions
-unset CARGO
-
-export XARGO="$bpf_sdk"/dependencies/bin/xargo
-export XARGO_TARGET=bpfel-unknown-unknown
-export XARGO_HOME="$bpf_sdk/dependencies/xargo"
-export XARGO_RUST_SRC="$bpf_sdk/dependencies/rust-bpf-sysroot/src"
-export RUST_COMPILER_RT_ROOT="$bpf_sdk/dependencies/rust-bpf-sysroot/src/compiler-rt"
--- a/sdk/bpf/rust/xargo-build.sh
+++ b/sdk/bpf/rust/xargo-build.sh
@ -1,29 +0,0 @@
-#!/usr/bin/env bash
-
-bpf_sdk=$(cd "$(dirname "$0")/.." && pwd)
-# shellcheck source=sdk/bpf/env.sh
-source "$bpf_sdk"/env.sh
-
-set -e
-(
-  while true; do
-    if [[ -r Xargo.toml ]]; then
-      break;
-    fi
-    if [[ $PWD = / ]]; then
-      cat <<EOF
-Error: Failed to find Xargo.toml
-
-Please create a Xargo.toml file in the same directory as your Cargo.toml with
-the following contents:
-
-  [target.bpfel-unknown-unknown.dependencies.std]
-  features = []
-
-EOF
-      exit 1
-    fi
-    cd ..
-  done
-)
-exec "$XARGO" build --target "$XARGO_TARGET" --release "$@"
--- a/sdk/bpf/scripts/install.sh
+++ b/sdk/bpf/scripts/install.sh
@ -40,24 +40,6 @@ download() {
  return 1
 }

-clone() {
-  declare url=$1
-  declare version=$2
-
-  rm -rf temp
-  if (
-    set -x
-    git clone --recursive --depth 1 --single-branch --branch "$version" "$url" temp
-  ); then
-    (
-      shopt -s dotglob nullglob
-      mv temp/* .
-    )
-    return 0
-  fi
-  return 1
-}
-
 get() {
  declare version=$1
  declare dirname=$2
@ -85,26 +67,6 @@ get() {
  return 1
 }

-# Install xargo
-version=0.3.22
-if [[ ! -e xargo-$version.md ]] || [[ ! -x bin/xargo ]]; then
-  (
-    args=()
-    # shellcheck disable=SC2154
-    if [[ -n $rust_stable ]]; then
-      args+=(+"$rust_stable")
-    fi
-    args+=(install xargo --version "$version" --root .)
-    set -ex
-    cargo "${args[@]}"
-  )
-  exitcode=$?
-  if [[ $exitcode -ne 0 ]]; then
-    exit 1
-  fi
-  ./bin/xargo --version >xargo-$version.md 2>&1
-fi
-
 # Install Criterion
 if [[ $machine == "linux" ]]; then
  version=v2.3.3
@ -130,7 +92,7 @@ if [[ ! -e criterion-$version.md || ! -e criterion ]]; then
 fi

 # Install Rust-BPF
-version=v1.1
+version=v1.2
 if [[ ! -e bpf-tools-$version.md || ! -e bpf-tools ]]; then
  (
    set -e
@ -148,31 +110,12 @@ if [[ ! -e bpf-tools-$version.md || ! -e bpf-tools ]]; then
    exit 1
  fi
  touch bpf-tools-$version.md
-fi
-set -ex
-./bpf-tools/rust/bin/rustc --print sysroot
-set +e
-rustup toolchain uninstall bpf
-set -e
-rustup toolchain link bpf bpf-tools/rust
-
-# Install Rust-BPF Sysroot sources
-version=v1.1
-if [[ ! -e rust-bpf-sysroot-$version.md || ! -e rust-bpf-sysroot ]]; then
-  (
-    set -e
-    rm -rf rust-bpf-sysroot*
-    rm -rf xargo
-    job="clone \
-           https://github.com/solana-labs/rust-bpf-sysroot.git \
-           $version"
-    get $version rust-bpf-sysroot "$job"
-  )
-  exitcode=$?
-  if [[ $exitcode -ne 0 ]]; then
-    exit 1
-  fi
-  touch rust-bpf-sysroot-$version.md
+  set -ex
+  ./bpf-tools/rust/bin/rustc --print sysroot
+  set +e
+  rustup toolchain uninstall bpf
+  set -e
+  rustup toolchain link bpf bpf-tools/rust
 fi

 exit 0
--- a/sdk/cargo-build-bpf/Cargo.toml
+++ b/sdk/cargo-build-bpf/Cargo.toml
@ -10,9 +10,12 @@ edition = "2018"
 publish = false

 [dependencies]
+bzip2 = "0.3.3"
 clap = "2.33.3"
 cargo_metadata = "0.12.0"
 solana-sdk = { path = "..", version = "=1.7.0" }
+solana-download-utils = { path = "../../download-utils", version = "=1.7.0" }
+tar = "0.4.28"

 [features]
 program = []
--- a/sdk/cargo-build-bpf/src/main.rs
+++ b/sdk/cargo-build-bpf/src/main.rs
@ -1,16 +1,20 @@
 use {
+    bzip2::bufread::BzDecoder,
    clap::{
        crate_description, crate_name, crate_version, value_t, value_t_or_exit, values_t, App, Arg,
    },
+    solana_download_utils::download_file,
    solana_sdk::signature::{write_keypair_file, Keypair},
    std::{
        env,
        ffi::OsStr,
-        fs,
+        fs::{self, File},
+        io::BufReader,
        path::{Path, PathBuf},
        process::exit,
-        process::Command,
+        process::{Command, Stdio},
    },
+    tar::Archive,
 };

 struct Config {
@ -44,7 +48,7 @@ impl Default for Config {
    }
 }

-fn spawn<I, S>(program: &Path, args: I)
+fn spawn<I, S>(program: &Path, args: I) -> String
 where
    I: IntoIterator<Item = S>,
    S: AsRef<OsStr>,
@ -56,18 +60,125 @@ where
    }
    println!();

-    let mut child = Command::new(program)
+    let child = Command::new(program)
        .args(&args)
+        .stdout(Stdio::piped())
        .spawn()
        .unwrap_or_else(|err| {
            eprintln!("Failed to execute {}: {}", program.display(), err);
            exit(1);
        });

-    let exit_status = child.wait().expect("failed to wait on child");
-    if !exit_status.success() {
+    let output = child.wait_with_output().expect("failed to wait on child");
+    if !output.status.success() {
        exit(1);
    }
+    output
+        .stdout
+        .as_slice()
+        .iter()
+        .map(|&c| c as char)
+        .collect::<String>()
+}
+
+// Check whether a package is installed and install it if missing.
+fn install_if_missing(
+    config: &Config,
+    package: &str,
+    version: &str,
+    url: &str,
+    file: &Path,
+) -> Result<(), String> {
+    // Check whether the package is already in ~/.cache/solana.
+    // Donwload it and place in the proper location if not found.
+    let home_dir = PathBuf::from(env::var("HOME").unwrap_or_else(|err| {
+        eprintln!("Can't get home directory path: {}", err);
+        exit(1);
+    }));
+    let target_path = home_dir
+        .join(".cache")
+        .join("solana")
+        .join(version)
+        .join(package);
+    if !target_path.is_dir() {
+        if target_path.exists() {
+            fs::remove_file(&target_path).map_err(|err| err.to_string())?;
+        }
+        let mut url = String::from(url);
+        url.push('/');
+        url.push_str(version);
+        url.push('/');
+        url.push_str(file.to_str().unwrap());
+        download_file(&url.as_str(), &file, false)?;
+        fs::create_dir_all(&target_path).map_err(|err| err.to_string())?;
+        let zip = File::open(&file).map_err(|err| err.to_string())?;
+        let tar = BzDecoder::new(BufReader::new(zip));
+        let mut archive = Archive::new(tar);
+        archive
+            .unpack(&target_path)
+            .map_err(|err| err.to_string())?;
+        fs::remove_file(file).map_err(|err| err.to_string())?;
+    }
+    // Make a symbolyc link source_path -> target_path in the
+    // sdk/bpf/dependencies directory if no valid link found.
+    let source_base = config.bpf_sdk.join("dependencies");
+    if !source_base.exists() {
+        fs::create_dir_all(&source_base).map_err(|err| err.to_string())?;
+    }
+    let source_path = source_base.join(package);
+    // Check whether the correct symbolic link exists.
+    let missing_source = if source_path.exists() {
+        let invalid_link = if let Ok(link_target) = source_path.read_link() {
+            link_target != target_path
+        } else {
+            true
+        };
+        if invalid_link {
+            fs::remove_file(&source_path).map_err(|err| err.to_string())?;
+        }
+        invalid_link
+    } else {
+        true
+    };
+    if missing_source {
+        #[cfg(unix)]
+        std::os::unix::fs::symlink(target_path, source_path).map_err(|err| err.to_string())?;
+        #[cfg(windows)]
+        std::os::windows::fs::symlink_dir(target_path, source_path)
+            .map_err(|err| err.to_string())?;
+    }
+    Ok(())
+}
+
+// check whether custom BPF toolchain is linked, and link it if it is not.
+fn link_bpf_toolchain(config: &Config) {
+    let toolchain_path = config
+        .bpf_sdk
+        .join("dependencies")
+        .join("bpf-tools")
+        .join("rust");
+    let rustup = PathBuf::from("rustup");
+    let rustup_args = vec!["toolchain", "list", "-v"];
+    let rustup_output = spawn(&rustup, &rustup_args);
+    let mut do_link = true;
+    for line in rustup_output.lines() {
+        if line.starts_with("bpf") {
+            let mut it = line.split_whitespace();
+            let _ = it.next();
+            let path = it.next();
+            if path.unwrap() != toolchain_path.to_str().unwrap() {
+                let rustup_args = vec!["toolchain", "uninstall", "bpf"];
+                spawn(&rustup, &rustup_args);
+            } else {
+                do_link = false;
+            }
+            break;
+        }
+    }
+    if do_link {
+        let rustup_args = vec!["toolchain", "link", "bpf", toolchain_path.to_str().unwrap()];
+        spawn(&rustup, &rustup_args);
+    }
 }

 fn build_bpf_package(config: &Config, target_directory: &Path, package: &cargo_metadata::Package) {
@ -141,27 +252,71 @@ fn build_bpf_package(config: &Config, target_directory: &Path, package: &cargo_m
    if legacy_program_feature_present {
        println!("Legacy program feature detected");
    }
+    let bpf_tools_filename = if cfg!(target_os = "macos") {
+        "solana-bpf-tools-osx.tar.bz2"
+    } else {
+        "solana-bpf-tools-linux.tar.bz2"
+    };
+    install_if_missing(
+        &config,
+        "bpf-tools",
+        "v1.2",
+        "https://github.com/solana-labs/bpf-tools/releases/download",
+        &PathBuf::from(bpf_tools_filename),
+    )
+    .expect("Failed to install bpf-tools");
+    link_bpf_toolchain(&config);

-    let xargo_build = config.bpf_sdk.join("rust").join("xargo-build.sh");
-    let mut xargo_build_args = vec![];
+    let llvm_bin = config
+        .bpf_sdk
+        .join("dependencies")
+        .join("bpf-tools")
+        .join("llvm")
+        .join("bin");
+    env::set_var("CC", llvm_bin.join("clang"));
+    env::set_var("AR", llvm_bin.join("llvm-ar"));
+    env::set_var("OBJDUMP", llvm_bin.join("llvm-objdump"));
+    env::set_var("OBJCOPY", llvm_bin.join("llvm-objcopy"));
+    let linker = llvm_bin.join("ld.lld");
+    let linker_script = config.bpf_sdk.join("rust").join("bpf.ld");
+    let mut rust_flags = String::from("-C lto=no");
+    rust_flags.push_str(" -C opt-level=2");
+    rust_flags.push_str(" -C link-arg=-z -C link-arg=notext");
+    rust_flags.push_str(" -C link-arg=-T");
+    rust_flags.push_str(linker_script.to_str().unwrap());
+    rust_flags.push_str(" -C link-arg=--Bdynamic");
+    rust_flags.push_str(" -C link-arg=-shared");
+    rust_flags.push_str(" -C link-arg=--threads=1");
+    rust_flags.push_str(" -C link-arg=--entry=entrypoint");
+    rust_flags.push_str(" -C linker=");
+    rust_flags.push_str(linker.to_str().unwrap());
+    env::set_var("RUSTFLAGS", rust_flags);

+    let cargo_build = PathBuf::from("cargo");
+    let mut cargo_build_args = vec![
+        "+bpf",
+        "build",
+        "--target",
+        "bpfel-unknown-unknown",
+        "--release",
+    ];
    if config.no_default_features {
-        xargo_build_args.push("--no-default-features");
+        cargo_build_args.push("--no-default-features");
    }
    for feature in &config.features {
-        xargo_build_args.push("--features");
-        xargo_build_args.push(feature);
+        cargo_build_args.push("--features");
+        cargo_build_args.push(feature);
    }
    if legacy_program_feature_present {
        if !config.no_default_features {
-            xargo_build_args.push("--no-default-features");
+            cargo_build_args.push("--no-default-features");
        }
-        xargo_build_args.push("--features=program");
+        cargo_build_args.push("--features=program");
    }
    if config.verbose {
-        xargo_build_args.push("--verbose");
+        cargo_build_args.push("--verbose");
    }
-    spawn(&config.bpf_sdk.join(xargo_build), &xargo_build_args);
+    spawn(&cargo_build, &cargo_build_args);

    if let Some(program_name) = program_name {
        let program_unstripped_so = target_build_directory.join(&format!("{}.so", program_name));
@ -257,6 +412,10 @@ fn build_bpf(config: Config, manifest_path: Option<PathBuf>) {
 }

 fn main() {
+    if cfg!(windows) {
+        println!("Solana Rust BPF toolchain is not available on Windows");
+        exit(1);
+    }
    let default_config = Config::default();
    let default_bpf_sdk = format!("{}", default_config.bpf_sdk.display());