diff --git a/Cargo.lock b/Cargo.lock index b15c791748..81ab0b6911 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -323,7 +323,7 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -836,7 +836,7 @@ dependencies = [ [[package]] name = "either" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1354,7 +1354,7 @@ name = "itertools" version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1362,7 +1362,7 @@ name = "itertools" version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2363,7 +2363,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", - "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "rayon-core 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3130,6 +3130,27 @@ dependencies = [ "ureq 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "solana-crate-features" +version = "0.19.0-pre0" +dependencies = [ + "backtrace 0.3.33 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)", + "curve25519-dalek 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", + "rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-ed25519-dalek 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "ureq 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "solana-drone" version = "0.19.0-pre0" @@ -3157,6 +3178,7 @@ dependencies = [ "curve25519-dalek 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -3553,6 +3575,7 @@ dependencies = [ "serde_derive 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "solana-crate-features 0.19.0-pre0", "solana-ed25519-dalek 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "solana-logger 0.19.0-pre0", "untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4540,7 +4563,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", - "either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -5144,7 +5167,7 @@ dependencies = [ "checksum docopt 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7f525a586d310c87df72ebcd98009e57f1cc030c8c268305287a476beb653969" "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e" "checksum ed25519-dalek 1.0.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)" = "81956bcf7ef761fb4e1d88de3fa181358a0d26cbcb9755b587a08f9119824b86" -"checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" +"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" "checksum elfkit 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "02f182eda16a7360c80a2f8638d0726e9d5478173058f1505c42536ca666ecd2" "checksum ena 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f56c93cc076508c549d9bb747f79aa9b4eb098be7b8cad8830c3137ef52d1e00" "checksum encode_unicode 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "90b2c9496c001e8cb61827acdefad780795c42264c137744cae6f7d9e3450abd" diff --git a/crate-features/.gitignore b/crate-features/.gitignore new file mode 100644 index 0000000000..5404b132db --- /dev/null +++ b/crate-features/.gitignore @@ -0,0 +1,2 @@ +/target/ +/farf/ diff --git a/crate-features/0001-Print-package-features.patch b/crate-features/0001-Print-package-features.patch new file mode 100644 index 0000000000..1c3b6a2051 --- /dev/null +++ b/crate-features/0001-Print-package-features.patch @@ -0,0 +1,44 @@ +From 64514728f549719b8b42af1570404d5f51730e8c Mon Sep 17 00:00:00 2001 +From: Michael Vines +Date: Fri, 13 Sep 2019 17:34:43 -0700 +Subject: [PATCH] Print package features + +--- + src/cargo/core/compiler/context/compilation_files.rs | 8 ++++++++ + src/cargo/core/compiler/fingerprint.rs | 2 +- + 2 files changed, 9 insertions(+), 1 deletion(-) + +diff --git a/src/cargo/core/compiler/context/compilation_files.rs b/src/cargo/core/compiler/context/compilation_files.rs +index ed3232ea..c98da4ca 100644 +--- a/src/cargo/core/compiler/context/compilation_files.rs ++++ b/src/cargo/core/compiler/context/compilation_files.rs +@@ -591,5 +591,13 @@ fn compute_metadata<'a, 'cfg>( + if let Ok(ref channel) = __cargo_default_lib_metadata { + channel.hash(&mut hasher); + } ++ ++ eprintln!( ++ "package {}: {} #{} features={:?}", ++ unit.pkg.package_id(), ++ unit.target, ++ hasher.finish(), ++ bcx.resolve.features_sorted(unit.pkg.package_id()), ++ ); + Some(Metadata(hasher.finish())) + } +diff --git a/src/cargo/core/compiler/fingerprint.rs b/src/cargo/core/compiler/fingerprint.rs +index 3738bcdd..de78ffb4 100644 +--- a/src/cargo/core/compiler/fingerprint.rs ++++ b/src/cargo/core/compiler/fingerprint.rs +@@ -830,7 +830,7 @@ impl Fingerprint { + // for a discussion of why it's `>` see the discussion about #5918 + // below in `find_stale`. + if dep_mtime > max_mtime { +- log::info!("dependency on `{}` is newer than we are", dep.name); ++ eprintln!("dependency on `{}` is newer than we are", dep.name); + return Ok(()); + } + } +-- +2.20.1 (Apple Git-117) + diff --git a/crate-features/Cargo.toml b/crate-features/Cargo.toml new file mode 100644 index 0000000000..46cd820450 --- /dev/null +++ b/crate-features/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "solana-crate-features" +version = "0.19.0-pre0" +authors = ["Solana Maintainers "] +repository = "https://github.com/solana-labs/solana" +homepage = "https://solana.com/" +license = "Apache-2.0" +edition = "2018" + +[dependencies] +backtrace = { version = "0.3.33", features = ["serialize-serde"] } +bytes = { version = "0.4.12", features = ["either"] } +cc = { version = "1.0.45", features = ["jobserver", "num_cpus", "parallel"]} +curve25519-dalek = { version = "1.1.3" } +either= { version = "1.5.3" } +failure = { version = "0.1.5" } +lazy_static = { version = "1.4.0", features = ["spin", "spin_no_std"] } +libc = { version = "0.2.62", features = ["extra_traits"] } +rand_chacha = { version = "0.1.1" } +regex-syntax = { version = "0.6.12" } +serde = { version = "1.0.100", features = ["rc"] } +solana-ed25519-dalek = { version = "0.2.0", features = ["serde"] } +syn = { version = "1.0.3", features = ["extra-traits", "full"] } +ureq = { version = "0.11.0", features = ["json"] } +winapi = { version = "0.3.8", features=["basetsd", "consoleapi", "errhandlingapi", "fileapi", "handleapi", "impl-debug", "impl-default", "knownfolders", "libloaderapi", "memoryapi", "minwinbase", "minwindef", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "utilapiset", "winbase", "wincon", "windef", "winerror", "winnls", "winnt", "winreg", "winsock2", "winuser", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] } diff --git a/crate-features/README.md b/crate-features/README.md new file mode 100644 index 0000000000..4d5c1aec43 --- /dev/null +++ b/crate-features/README.md @@ -0,0 +1,58 @@ + +Cargo (as of 1.37) does not nicely handle the scenario where features of a +dependent crate of multiple other crates in the tree vary. + +To illustrate the problem consider crates A, B and C arranged as follows: +* Crate A and B are both members of a Cargo virtual manifest +* Crate C provides two features, F1 and F2 +* Crate A requests feature F1 of C, crate B requests F2 of C + +When crate A and B are built together, `cargo` builds C with both feature F1 and +F2 enabled (the union of all enabled features). However when A or B are built +individually, `cargo` builds C with only feature F1 or F2 enabled. + +Unfortunately in all these cases, `cargo` will build crate C in the same target +location and the outputs for C will be recreated every time the features for crate C +change. + +From a clean workspace, building A individually first will cause C to be built +as expected. Then build B individually and C will be re-built because F2 was +enabled instead of F1. Now rebuild A and observe that C will be re-built again +because F1 was re-enabled. + +In practice this problem is much less obvious as both A and B likely to not have +a direct dependency on C, indirectly causing rebuilds of numerous other crates as well. + +The `solana-crate-features` offers a workaround to this "feature thrashing" +problem by explicitly declaring all "C-like crates" with the union of all features +that any other crate in the tree (either explicitly or implicitly) enable. All +crates in the Solana source tree should depend on `solana-crate-features`. + +### Adding new dependent crates +When unnecessary `cargo` rebuilds are observed, the first step is to figure what +dependent crate is suffering from feature thrashing. + +This information is not readily available from the stock `cargo` program, so use +the following steps to produce a custom `cargo` program that will output the +necessary feature information to stderr during a build: +```bash +$ git clone git@github.com:rust-lang/cargo.git -b rust-1.37.0 +$ cd cargo +$ git apply 0001-Print-package-features.patch +$ cargo build +$ mv ~/.cargo/bin/cargo ~/.cargo/bin/cargo.org +$ cp ./target/debug/cargo ~/.cargo/bin/cargo +``` + +Rebuild with the custom `cargo` and search for indications of crates getting +built with different features (repeated runs of `./scripts/cargo-install-all.sh` +work great for this). When the problematic crate is identified, add it as a +dependency of `solana-crate-features` with the union of all observed enabled +features for that crate. + +### Appendix +This command will enable some additional cargo log output that can help debug +dependency problems as well: +```bash +export CARGO_LOG=cargo::core::compiler::fingerprint=info +``` diff --git a/crate-features/src/lib.rs b/crate-features/src/lib.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/crate-features/src/lib.rs @@ -0,0 +1 @@ + diff --git a/multinode-demo/common.sh b/multinode-demo/common.sh index 97d9aa79b4..e634c3a327 100644 --- a/multinode-demo/common.sh +++ b/multinode-demo/common.sh @@ -55,7 +55,7 @@ else if [[ -n $NDEBUG ]]; then maybe_release=--release fi - declare manifest_path="--manifest-path=$SOLANA_ROOT/$crate/Cargo.toml" + declare manifest_path="--manifest-path=$SOLANA_ROOT/Cargo.toml" printf "cargo run $manifest_path $maybe_release $maybe_package --bin %s %s -- " "$program" "$features" } fi diff --git a/net/net.sh b/net/net.sh index 9864e39334..05f516b7d1 100755 --- a/net/net.sh +++ b/net/net.sh @@ -125,7 +125,6 @@ internalNodesLamports= maybeNoSnapshot="" maybeSkipLedgerVerify="" maybeDisableAirdrops="" -buildProfile="--release" debugBuild=false doBuild=true @@ -276,8 +275,6 @@ while getopts "h?T:t:o:f:rD:c:Fn:i:d" opt "${shortArgs[@]}"; do esac done -$debugBuild && buildProfile="" - loadConfigFile if [[ -n $numFullnodesRequested ]]; then @@ -332,9 +329,15 @@ build() { # shellcheck source=/dev/null source target/perf-libs/env.sh fi + + buildVariant= + if $debugBuild; then + buildVariant=debug + fi + $MAYBE_DOCKER bash -c " set -ex - scripts/cargo-install-all.sh farf \"$cargoFeatures\" \"$buildProfile\" + scripts/cargo-install-all.sh farf \"$cargoFeatures\" \"$buildVariant\" if [[ -n \"$customPrograms\" ]]; then scripts/cargo-install-custom-programs.sh farf $customPrograms fi diff --git a/scripts/cargo-install-all.sh b/scripts/cargo-install-all.sh index 5348376c19..48489d35eb 100755 --- a/scripts/cargo-install-all.sh +++ b/scripts/cargo-install-all.sh @@ -16,16 +16,18 @@ if [[ -z $1 ]]; then fi installDir="$(mkdir -p "$1"; cd "$1"; pwd)" +cargo=cargo cargoFeatures="$2" +debugBuild="$3" -buildProfile="$3" -if [[ -n "$buildProfile" ]]; then - binTargetDir="release" -else - binTargetDir="debug" +buildVariant=release +maybeReleaseFlag=--release +if [[ -n "$debugBuild" ]]; then + maybeReleaseFlag= + buildVariant=debug fi -echo "Install location: $installDir" +echo "Install location: $installDir ($buildVariant)" cd "$(dirname "$0")"/.. @@ -34,36 +36,45 @@ SECONDS=0 ( set -x # shellcheck disable=SC2086 # Don't want to double quote $rust_version - cargo $rust_version build --all $buildProfile --features="$cargoFeatures" + $cargo $rust_version build --all $maybeReleaseFlag --features="$cargoFeatures" ) -BIN_CRATES=( - drone - gossip - install - keygen - ledger-tool - replicator - validator - cli - bench-exchange - bench-tps +BINS=( + solana-drone + solana-gossip + solana-install + solana-install-init + solana-keygen + solana-ledger-tool + solana-replicator + solana-validator + solana + solana-bench-exchange + solana-bench-tps ) #XXX: Ensure `solana-genesis` is built LAST! # See https://github.com/solana-labs/solana/issues/5826 -BIN_CRATES+=( "genesis" ) +BINS+=(solana-genesis) -for crate in "${BIN_CRATES[@]}"; do - ( - set -x - # shellcheck disable=SC2086 # Don't want to double quote $rust_version - cargo $rust_version install --force --path "$crate" --root "$installDir" --features="$cargoFeatures" - ) +binArgs=() +for bin in "${BINS[@]}"; do + binArgs+=(--bin "$bin") +done + +( + set -x + # shellcheck disable=SC2086 # Don't want to double quote $rust_version + $cargo $rust_version build $maybeReleaseFlag "${binArgs[@]}" --features="$cargoFeatures" +) + +mkdir -p "$installDir/bin" +for bin in "${BINS[@]}"; do + cp -fv "target/$buildVariant/$bin" "$installDir"/bin done for dir in programs/*; do - for program in echo target/$binTargetDir/deps/libsolana_"$(basename "$dir")".{so,dylib,dll}; do + for program in echo target/$buildVariant/deps/libsolana_"$(basename "$dir")".{so,dylib,dll}; do if [[ -f $program ]]; then mkdir -p "$installDir/bin/deps" rm -f "$installDir/bin/deps/$(basename "$program")" @@ -72,5 +83,4 @@ for dir in programs/*; do done done -du -a "$installDir" echo "Done after $SECONDS seconds" diff --git a/sdk/Cargo.toml b/sdk/Cargo.toml index cb7ba7d60c..23285afd59 100644 --- a/sdk/Cargo.toml +++ b/sdk/Cargo.toml @@ -23,6 +23,7 @@ default = [ "solana-ed25519-dalek", "solana-logger", "untrusted", + "solana-crate-features" ] [dependencies] @@ -46,4 +47,5 @@ serde_json = { version = "1.0.40", optional = true } sha2 = "0.8.0" solana-ed25519-dalek = { version = "0.2.0", optional = true } solana-logger = { path = "../logger", version = "0.19.0-pre0", optional = true } -untrusted = { version = "0.7.0", optional = true } \ No newline at end of file +untrusted = { version = "0.7.0", optional = true } +solana-crate-features = { path = "../crate-features", version = "0.19.0-pre0", optional = true }