Add cargo-build-bpf

Cargo.lock

@@ -440,6 +440,17 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "cargo_metadata"
+version = "0.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5a5f7b42f606b7f23674f6f4d877628350682bc40687d3fae65679a58d55345"
+dependencies = [
+ "semver 0.11.0",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "cast"
 version = "0.2.3"
@@ -499,9 +510,9 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "2.33.1"
+version = "2.33.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129"
+checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
 dependencies = [
  "ansi_term",
  "atty",
@@ -3180,6 +3191,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6"
 dependencies = [
  "semver-parser 0.10.1",
+ "serde",
 ]
 
 [[package]]
@@ -3238,9 +3250,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.56"
+version = "1.0.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3"
+checksum = "dcac07dbffa1c65e7f816ab9eba78eb142c6d44410f4eeba1e26e4f5dfa56b95"
 dependencies = [
  "itoa",
  "ryu",
@@ -3621,6 +3633,14 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "solana-cargo-build-bpf"
+version = "1.5.0"
+dependencies = [
+ "cargo_metadata",
+ "clap",
+]
+
 [[package]]
 name = "solana-clap-utils"
 version = "1.5.0"

Cargo.toml

@@ -53,6 +53,7 @@ members = [
     "ramp-tps",
     "runtime",
     "sdk",
+    "sdk/cargo-build-bpf",
     "scripts",
     "stake-accounts",
     "stake-monitor",

cargo-build-bpf

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+here=$(dirname "$0")
+set -x
+exec cargo run --manifest-path $here/sdk/cargo-build-bpf/Cargo.toml -- --bpf-sdk $here/sdk/bpf "$@"

ci/nits.sh

@@ -26,6 +26,7 @@ declare print_free_tree=(
   ':runtime/src/**.rs'
   ':sdk/bpf/rust/rust-utils/**.rs'
   ':sdk/**.rs'
+  ':^sdk/cargo-build-bpf/**.rs'
   ':^sdk/src/program_option.rs'
   ':^sdk/src/program_stubs.rs'
   ':programs/**.rs'

frozen-abi/Xargo.toml

@@ -0,0 +1,2 @@
+[target.bpfel-unknown-unknown.dependencies.std]
+features = []

scripts/cargo-install-all.sh

@@ -75,6 +75,7 @@ else
 
 
   BINS=(
+    cargo-build-bpf
     solana
     solana-bench-exchange
     solana-bench-tps

sdk/bpf/env.sh

@@ -0,0 +1,39 @@
+#
+# Configures the BPF SDK environment
+#
+
+if [ -z "$bpf_sdk" ]; then
+  bpf_sdk=.
+fi
+
+# Ensure the sdk is installed
+"$bpf_sdk"/scripts/install.sh
+
+# Use the SDK's version of llvm to build the compiler-builtins for BPF
+export CC="$bpf_sdk/dependencies/llvm-native/bin/clang"
+export AR="$bpf_sdk/dependencies/llvm-native/bin/llvm-ar"
+export OBJDUMP="$bpf_sdk/dependencies/llvm-native/bin/llvm-objdump"
+export OBJCOPY="$bpf_sdk/dependencies/llvm-native/bin/llvm-objcopy"
+
+# Use the SDK's version of Rust to build for BPF
+export RUSTUP_TOOLCHAIN=bpf
+export RUSTFLAGS="
+    -C lto=no \
+    -C opt-level=2 \
+    -C link-arg=-z -C link-arg=notext \
+    -C link-arg=-T$bpf_sdk/rust/bpf.ld \
+    -C link-arg=--Bdynamic \
+    -C link-arg=-shared \
+    -C link-arg=--entry=entrypoint \
+    -C link-arg=-no-threads \
+    -C linker=$bpf_sdk/dependencies/llvm-native/bin/ld.lld"
+
+# CARGO may be set if run from within cargo, causing
+# incompatibilities between cargo and xargo versions
+unset CARGO
+
+export XARGO="$bpf_sdk"/dependencies/bin/xargo
+export XARGO_TARGET=bpfel-unknown-unknown
+export XARGO_HOME="$bpf_sdk/dependencies/xargo"
+export XARGO_RUST_SRC="$bpf_sdk/dependencies/rust-bpf-sysroot/src"
+export RUST_COMPILER_RT_ROOT="$bpf_sdk/dependencies/rust-bpf-sysroot/src/compiler-rt"

sdk/bpf/rust/build.sh

@@ -1,49 +1,21 @@
 #!/usr/bin/env bash
 
-if [ "$#" -ne 1 ]; then
+if [[ "$#" -ne 1 ]]; then
     echo "Error: Must provide the full path to the project to build"
     exit 1
 fi
-if [ ! -f "$1/Cargo.toml" ]; then
+if [[ ! -f "$1/Cargo.toml" ]]; then
       echo "Error: Cannot find project: $1"
     exit 1
 fi
+bpf_sdk=$(cd "$(dirname "$0")/.." && pwd)
 
 echo "Building $1"
 set -e
-
-bpf_sdk=$(cd "$(dirname "$0")/.." && pwd)
-
-# Ensure the sdk is installed
-"$bpf_sdk"/scripts/install.sh
-
-# Use the SDK's version of llvm to build the compiler-builtins for BPF
-export CC="$bpf_sdk/dependencies/llvm-native/bin/clang"
-export AR="$bpf_sdk/dependencies/llvm-native/bin/llvm-ar"
-
-# Use the SDK's version of Rust to build for BPF
-export RUSTUP_TOOLCHAIN=bpf
-export RUSTFLAGS="
-    -C lto=no \
-    -C opt-level=2 \
-    -C link-arg=-z -C link-arg=notext \
-    -C link-arg=-T$bpf_sdk/rust/bpf.ld \
-    -C link-arg=--Bdynamic \
-    -C link-arg=-shared \
-    -C link-arg=--entry=entrypoint \
-    -C link-arg=-no-threads \
-    -C linker=$bpf_sdk/dependencies/llvm-native/bin/ld.lld"
-
-# CARGO may be set if build.sh is run from within cargo, causing
-# incompatibilities between cargo and xargo versions
-unset CARGO
-
-# Setup xargo
-export XARGO_HOME="$bpf_sdk/dependencies/xargo"
-export XARGO_RUST_SRC="$bpf_sdk/dependencies/rust-bpf-sysroot/src"
-export RUST_COMPILER_RT_ROOT="$bpf_sdk/dependencies/rust-bpf-sysroot/src/compiler-rt"
+# shellcheck source=sdk/bpf/env.sh
+source "$bpf_sdk"/env.sh
 
 cd "$1"
-xargo build --target bpfel-unknown-unknown --release --no-default-features --features program
+"$XARGO" build --target "$XARGO_TARGET" --release --no-default-features --features program
 
 { { set +x; } 2>/dev/null; echo Success; }

sdk/bpf/rust/xargo-build.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+bpf_sdk=$(cd "$(dirname "$0")/.." && pwd)
+# shellcheck source=sdk/bpf/env.sh
+source "$bpf_sdk"/env.sh
+
+set -e
+(
+  while true; do
+    if [[ -r Xargo.toml ]]; then
+      break;
+    fi
+    if [[ $PWD = / ]]; then
+      cat <<EOF
+Error: Failed to find Xargo.toml
+
+Please create a Xargo.toml file in the same directory as your Cargo.toml with
+the following contents:
+
+  [target.bpfel-unknown-unknown.dependencies.std]
+  features = []
+
+EOF
+      exit 1
+    fi
+    cd ..
+  done
+)
+exec "$XARGO" build --target "$XARGO_TARGET" --release "$@"

sdk/bpf/scripts/dump.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+bpf_sdk=$(cd "$(dirname "$0")/.." && pwd)
+# shellcheck source=sdk/bpf/env.sh
+source "$bpf_sdk"/env.sh
+
+so=$1
+dump=$2
+if [[ -z $so ]] || [[ -z $dump ]]; then
+  echo "Usage: $0 bpf-program.so dump.txt"
+  exit 1
+fi
+
+if [[ ! -r $so ]]; then
+  echo "Error: File not found or readable: $so"
+  exit 1
+fi
+
+if ! command -v rustfilt > /dev/null; then
+  echo "Error: rustfilt not found.  It can be installed by running: cargo install rustfilt"
+  exit 1
+fi
+if ! command -v readelf > /dev/null; then
+  if [[ $(uname) = Darwin ]]; then
+      echo "Error: readelf not found.  It can be installed by running: brew install binutils"
+  else
+      echo "Error: readelf not found."
+  fi
+  exit 1
+fi
+
+dump_mangled=$dump.mangled
+
+(
+  set -ex
+  ls -la "$so" > "$dump_mangled"
+  readelf -aW "$so" >>"$dump_mangled"
+  "$OBJDUMP" -print-imm-hex --source --disassemble "$so" >> "$dump_mangled"
+  sed s/://g < "$dump_mangled" | rustfilt > "$dump"
+)
+rm -f "$dump_mangled"
+
+if [[ ! -f "$dump" ]]; then
+  echo "Error: Failed to create $dump"
+  exit 1
+fi
+
+echo "Wrote $dump"

sdk/bpf/scripts/install.sh

@@ -84,19 +84,24 @@ clone() {
 }
 
 # Install xargo
-(
-  set -ex
-  # shellcheck disable=SC2154
-  if [[ -n $rust_stable ]]; then
-    cargo +"$rust_stable" install xargo
-  else
-    cargo install xargo
+version=0.3.22
+if [[ ! -e xargo-$version.md ]] || [[ ! -x bin/xargo ]]; then
+  (
+    args=()
+    # shellcheck disable=SC2154
+    if [[ -n $rust_stable ]]; then
+      args+=(+"$rust_stable")
+    fi
+    args+=(install xargo --version "$version" --root .)
+    set -ex
+    cargo "${args[@]}"
+    ./bin/xargo --version >xargo-$version.md 2>&1
+  )
+  exitcode=$?
+  if [[ $exitcode -ne 0 ]]; then
+    rm -rf xargo-$version.md
+    exit 1
   fi
-  xargo --version >xargo.md 2>&1
-)
-# shellcheck disable=SC2181
-if [[ $? -ne 0 ]]; then
-  exit 1
 fi
 
 # Install Criterion

sdk/bpf/scripts/objcopy.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+bpf_sdk=$(cd "$(dirname "$0")/.." && pwd)
+# shellcheck source=sdk/bpf/env.sh
+source "$bpf_sdk"/env.sh
+exec "$bpf_sdk"/dependencies/llvm-native/bin/llvm-objcopy "$@"

sdk/bpf/scripts/strip.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+so=$1
+if [[ ! -r $so ]]; then
+  echo "Error: file not found: $so"
+  exit 1
+fi
+so_stripped=$2
+if [[ -z $so_stripped ]]; then
+  echo "Usage: $0 unstripped.so stripped.so"
+  exit 1
+fi
+
+bpf_sdk=$(cd "$(dirname "$0")/.." && pwd)
+# shellcheck source=sdk/bpf/env.sh
+source "$bpf_sdk"/env.sh
+"$bpf_sdk"/dependencies/llvm-native/bin/llvm-objcopy --strip-all "$so" "$so_stripped"

sdk/cargo-build-bpf/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "solana-cargo-build-bpf"
+version = "1.5.0"
+description = "Compile a local package and all of its dependencies using the Solana BPF SDK"
+authors = ["Solana Maintainers <maintainers@solana.foundation>"]
+repository = "https://github.com/solana-labs/solana"
+homepage = "https://solana.com/"
+license = "Apache-2.0"
+edition = "2018"
+
+[dependencies]
+clap = "2.33.3"
+cargo_metadata = "0.12.0"
+
+[features]
+program = []
+
+[[bin]]
+name = "cargo-build-bpf"
+path = "src/main.rs"

sdk/cargo-build-bpf/src/main.rs

@@ -0,0 +1,258 @@
+use clap::{
+    crate_description, crate_name, crate_version, value_t, value_t_or_exit, values_t, App, Arg,
+};
+use std::{
+    env,
+    ffi::OsStr,
+    fs,
+    path::{Path, PathBuf},
+    process::exit,
+    process::Command,
+};
+
+struct Config {
+    bpf_sdk: PathBuf,
+    dump: bool,
+    features: Vec<String>,
+    manifest_path: Option<PathBuf>,
+    no_default_features: bool,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            bpf_sdk: env::current_exe()
+                .expect("Unable to get current directory")
+                .parent()
+                .expect("Unable to get parent directory")
+                .to_path_buf()
+                .join("sdk/bpf"),
+            features: vec![],
+            manifest_path: None,
+            no_default_features: false,
+            dump: true,
+        }
+    }
+}
+
+fn spawn<I, S>(program: &Path, args: I)
+where
+    I: IntoIterator<Item = S>,
+    S: AsRef<OsStr>,
+{
+    let args = args.into_iter().collect::<Vec<_>>();
+    print!("Running: {}", program.display());
+    for arg in args.iter() {
+        print!(" {}", arg.as_ref().to_str().unwrap_or("?"));
+    }
+    println!();
+
+    let mut child = Command::new(program)
+        .args(&args)
+        .spawn()
+        .unwrap_or_else(|err| {
+            eprintln!("Failed to execute {}: {}", program.display(), err);
+            exit(1);
+        });
+
+    let exit_status = child.wait().expect("failed to wait on child");
+    if !exit_status.success() {
+        exit(1);
+    }
+}
+
+fn build_bpf(config: Config) {
+    let mut metadata_command = cargo_metadata::MetadataCommand::new();
+    if let Some(manifest_path) = config.manifest_path {
+        metadata_command.manifest_path(manifest_path);
+    }
+
+    let metadata = metadata_command.exec().unwrap_or_else(|err| {
+        eprintln!("Failed to obtain package metadata: {}", err);
+        exit(1);
+    });
+
+    let root_package = metadata.root_package().unwrap_or_else(|| {
+        eprintln!(
+            "Workspace does not have a root package: {}",
+            metadata.workspace_root.display()
+        );
+        exit(1);
+    });
+
+    let program_name = {
+        let cdylib_targets = root_package
+            .targets
+            .iter()
+            .filter_map(|target| {
+                if target.crate_types.contains(&"cdylib".to_string()) {
+                    Some(&target.name)
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+
+        match cdylib_targets.len() {
+            0 => {
+                println!(
+                    "Note: {} crate does not contain a cdylib target",
+                    root_package.name
+                );
+                None
+            }
+            1 => Some(cdylib_targets[0].replace("-", "_")),
+            _ => {
+                eprintln!(
+                    "{} crate contains multiple cdylib targets: {:?}",
+                    root_package.name, cdylib_targets
+                );
+                exit(1);
+            }
+        }
+    };
+
+    let legacy_program_feature_present = root_package.features.contains_key("program");
+    let root_package_dir = &root_package.manifest_path.parent().unwrap_or_else(|| {
+        eprintln!(
+            "Unable to get directory of {}",
+            root_package.manifest_path.display()
+        );
+        exit(1);
+    });
+
+    let target_build_directory = metadata
+        .target_directory
+        .join("bpfel-unknown-unknown/release");
+
+    env::set_current_dir(&root_package_dir).unwrap_or_else(|err| {
+        eprintln!(
+            "Unable to set current directory to {}: {}",
+            root_package_dir.display(),
+            err
+        );
+        exit(1);
+    });
+
+    println!("BPF SDK: {}", config.bpf_sdk.display());
+    if config.no_default_features {
+        println!("No default features");
+    }
+    if !config.features.is_empty() {
+        println!("Features: {}", config.features.join(" "));
+    }
+    if legacy_program_feature_present {
+        println!("Legacy program feature detected");
+    }
+
+    let xargo_build = config.bpf_sdk.join("rust/xargo-build.sh");
+    let mut spawn_args = vec![];
+
+    if config.no_default_features {
+        spawn_args.push("--no-default-features");
+    }
+    for feature in &config.features {
+        spawn_args.push("--features");
+        spawn_args.push(feature);
+    }
+    if legacy_program_feature_present {
+        if !config.no_default_features {
+            spawn_args.push("--no-default-features");
+        }
+        spawn_args.push("--features=program");
+    }
+    spawn(&config.bpf_sdk.join(xargo_build), &spawn_args);
+
+    if let Some(program_name) = program_name {
+        let program_unstripped_so = target_build_directory.join(&format!("{}.so", program_name));
+        let program_dump = PathBuf::from(format!("{}-dump.txt", program_name));
+        let program_so = PathBuf::from(format!("{}.so", program_name));
+
+        spawn(
+            &config.bpf_sdk.join("scripts/strip.sh"),
+            &[&program_unstripped_so, &program_so],
+        );
+
+        if config.dump {
+            spawn(
+                &config.bpf_sdk.join("scripts/dump.sh"),
+                &[&program_unstripped_so, &program_dump],
+            );
+        }
+    } else if config.dump {
+        println!("Note: --dump is only available for crates with a cdylib target");
+    }
+}
+
+fn main() {
+    let default_bpf_sdk = format!("{}", Config::default().bpf_sdk.display());
+
+    let mut args = env::args().collect::<Vec<_>>();
+    // When run as a cargo subcommand, the first program argument is the subcommand name.
+    // Remove it
+    if let Some(arg1) = args.get(1) {
+        if arg1 == "build-bpf" {
+            args.remove(1);
+        }
+    }
+
+    let matches = App::new(crate_name!())
+        .about(crate_description!())
+        .version(crate_version!())
+        .arg(
+            Arg::with_name("bpf_sdk")
+                .long("bpf-sdk")
+                .value_name("PATH")
+                .takes_value(true)
+                .default_value(&default_bpf_sdk)
+                .help("Path to the Solana BPF SDK"),
+        )
+        .arg(
+            Arg::with_name("dump")
+                .long("dump")
+                .takes_value(false)
+                .help("Dump ELF information to a text file on success"),
+        )
+        .arg(
+            Arg::with_name("features")
+                .long("features")
+                .value_name("FEATURES")
+                .takes_value(true)
+                .multiple(true)
+                .help("Space-separated list of features to activate"),
+        )
+        .arg(
+            Arg::with_name("no_default_features")
+                .long("no-default-features")
+                .takes_value(false)
+                .help("Do not activate the `default` feature"),
+        )
+        .arg(
+            Arg::with_name("manifest_path")
+                .long("manifest-path")
+                .value_name("PATH")
+                .takes_value(true)
+                .help("Path to Cargo.toml"),
+        )
+        .get_matches_from(args);
+
+    let bpf_sdk = value_t_or_exit!(matches, "bpf_sdk", PathBuf);
+
+    let config = Config {
+        bpf_sdk: fs::canonicalize(&bpf_sdk).unwrap_or_else(|err| {
+            eprintln!(
+                "BPF SDK path does not exist: {}: {}",
+                bpf_sdk.display(),
+                err
+            );
+            exit(1);
+        }),
+        dump: matches.is_present("dump"),
+        features: values_t!(matches, "features", String)
+            .ok()
+            .unwrap_or_else(Vec::new),
+        manifest_path: value_t!(matches, "manifest_path", PathBuf).ok(),
+        no_default_features: matches.is_present("no_default_features"),
+    };
+    build_bpf(config);
+}