Add solana-crate-features workaround to avoid cargo feature thrashing (#5904)
automerge
This commit is contained in:
Michael Vines
Grimes
parent
8135279335
commit
e1f4e8a84a
|
|
@ -323,7 +323,7 @@ version = "0.4.12"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"iovec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
|
|
@ -836,7 +836,7 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.5.2"
|
||||
version = "1.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1354,7 +1354,7 @@ name = "itertools"
|
|||
version = "0.7.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1362,7 +1362,7 @@ name = "itertools"
|
|||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2363,7 +2363,7 @@ version = "1.2.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"crossbeam-deque 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rayon-core 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
|
|
@ -3130,6 +3130,27 @@ dependencies = [
|
|||
"ureq 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-crate-features"
|
||||
version = "0.19.0-pre0"
|
||||
dependencies = [
|
||||
"backtrace 0.3.33 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"cc 1.0.45 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"curve25519-dalek 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand_chacha 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"regex-syntax 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-ed25519-dalek 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"syn 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"ureq 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "solana-drone"
|
||||
version = "0.19.0-pre0"
|
||||
|
|
@ -3157,6 +3178,7 @@ dependencies = [
|
|||
"curve25519-dalek 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
|
|
@ -3553,6 +3575,7 @@ dependencies = [
|
|||
"serde_derive 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-crate-features 0.19.0-pre0",
|
||||
"solana-ed25519-dalek 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"solana-logger 0.19.0-pre0",
|
||||
"untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
|
|
@ -4540,7 +4563,7 @@ version = "0.1.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"futures 0.1.28 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
|
|
@ -5144,7 +5167,7 @@ dependencies = [
|
|||
"checksum docopt 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7f525a586d310c87df72ebcd98009e57f1cc030c8c268305287a476beb653969"
|
||||
"checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e"
|
||||
"checksum ed25519-dalek 1.0.0-pre.1 (registry+https://github.com/rust-lang/crates.io-index)" = "81956bcf7ef761fb4e1d88de3fa181358a0d26cbcb9755b587a08f9119824b86"
|
||||
"checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b"
|
||||
"checksum either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
|
||||
"checksum elfkit 0.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "02f182eda16a7360c80a2f8638d0726e9d5478173058f1505c42536ca666ecd2"
|
||||
"checksum ena 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f56c93cc076508c549d9bb747f79aa9b4eb098be7b8cad8830c3137ef52d1e00"
|
||||
"checksum encode_unicode 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "90b2c9496c001e8cb61827acdefad780795c42264c137744cae6f7d9e3450abd"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
/target/
|
||||
/farf/
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
From 64514728f549719b8b42af1570404d5f51730e8c Mon Sep 17 00:00:00 2001
|
||||
From: Michael Vines <mvines@gmail.com>
|
||||
Date: Fri, 13 Sep 2019 17:34:43 -0700
|
||||
Subject: [PATCH] Print package features
|
||||
|
||||
---
|
||||
src/cargo/core/compiler/context/compilation_files.rs | 8 ++++++++
|
||||
src/cargo/core/compiler/fingerprint.rs | 2 +-
|
||||
2 files changed, 9 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/src/cargo/core/compiler/context/compilation_files.rs b/src/cargo/core/compiler/context/compilation_files.rs
|
||||
index ed3232ea..c98da4ca 100644
|
||||
--- a/src/cargo/core/compiler/context/compilation_files.rs
|
||||
+++ b/src/cargo/core/compiler/context/compilation_files.rs
|
||||
@@ -591,5 +591,13 @@ fn compute_metadata<'a, 'cfg>(
|
||||
if let Ok(ref channel) = __cargo_default_lib_metadata {
|
||||
channel.hash(&mut hasher);
|
||||
}
|
||||
+
|
||||
+ eprintln!(
|
||||
+ "package {}: {} #{} features={:?}",
|
||||
+ unit.pkg.package_id(),
|
||||
+ unit.target,
|
||||
+ hasher.finish(),
|
||||
+ bcx.resolve.features_sorted(unit.pkg.package_id()),
|
||||
+ );
|
||||
Some(Metadata(hasher.finish()))
|
||||
}
|
||||
diff --git a/src/cargo/core/compiler/fingerprint.rs b/src/cargo/core/compiler/fingerprint.rs
|
||||
index 3738bcdd..de78ffb4 100644
|
||||
--- a/src/cargo/core/compiler/fingerprint.rs
|
||||
+++ b/src/cargo/core/compiler/fingerprint.rs
|
||||
@@ -830,7 +830,7 @@ impl Fingerprint {
|
||||
// for a discussion of why it's `>` see the discussion about #5918
|
||||
// below in `find_stale`.
|
||||
if dep_mtime > max_mtime {
|
||||
- log::info!("dependency on `{}` is newer than we are", dep.name);
|
||||
+ eprintln!("dependency on `{}` is newer than we are", dep.name);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
--
|
||||
2.20.1 (Apple Git-117)
|
||||
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
[package]
|
||||
name = "solana-crate-features"
|
||||
version = "0.19.0-pre0"
|
||||
authors = ["Solana Maintainers <maintainers@solana.com>"]
|
||||
repository = "https://github.com/solana-labs/solana"
|
||||
homepage = "https://solana.com/"
|
||||
license = "Apache-2.0"
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
backtrace = { version = "0.3.33", features = ["serialize-serde"] }
|
||||
bytes = { version = "0.4.12", features = ["either"] }
|
||||
cc = { version = "1.0.45", features = ["jobserver", "num_cpus", "parallel"]}
|
||||
curve25519-dalek = { version = "1.1.3" }
|
||||
either= { version = "1.5.3" }
|
||||
failure = { version = "0.1.5" }
|
||||
lazy_static = { version = "1.4.0", features = ["spin", "spin_no_std"] }
|
||||
libc = { version = "0.2.62", features = ["extra_traits"] }
|
||||
rand_chacha = { version = "0.1.1" }
|
||||
regex-syntax = { version = "0.6.12" }
|
||||
serde = { version = "1.0.100", features = ["rc"] }
|
||||
solana-ed25519-dalek = { version = "0.2.0", features = ["serde"] }
|
||||
syn = { version = "1.0.3", features = ["extra-traits", "full"] }
|
||||
ureq = { version = "0.11.0", features = ["json"] }
|
||||
winapi = { version = "0.3.8", features=["basetsd", "consoleapi", "errhandlingapi", "fileapi", "handleapi", "impl-debug", "impl-default", "knownfolders", "libloaderapi", "memoryapi", "minwinbase", "minwindef", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "processthreadsapi", "profileapi", "shlobj", "std", "synchapi", "sysinfoapi", "timezoneapi", "utilapiset", "winbase", "wincon", "windef", "winerror", "winnls", "winnt", "winreg", "winsock2", "winuser", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] }
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
|
||||
Cargo (as of 1.37) does not nicely handle the scenario where features of a
|
||||
dependent crate of multiple other crates in the tree vary.
|
||||
|
||||
To illustrate the problem consider crates A, B and C arranged as follows:
|
||||
* Crate A and B are both members of a Cargo virtual manifest
|
||||
* Crate C provides two features, F1 and F2
|
||||
* Crate A requests feature F1 of C, crate B requests F2 of C
|
||||
|
||||
When crate A and B are built together, `cargo` builds C with both feature F1 and
|
||||
F2 enabled (the union of all enabled features). However when A or B are built
|
||||
individually, `cargo` builds C with only feature F1 or F2 enabled.
|
||||
|
||||
Unfortunately in all these cases, `cargo` will build crate C in the same target
|
||||
location and the outputs for C will be recreated every time the features for crate C
|
||||
change.
|
||||
|
||||
From a clean workspace, building A individually first will cause C to be built
|
||||
as expected. Then build B individually and C will be re-built because F2 was
|
||||
enabled instead of F1. Now rebuild A and observe that C will be re-built again
|
||||
because F1 was re-enabled.
|
||||
|
||||
In practice this problem is much less obvious as both A and B likely to not have
|
||||
a direct dependency on C, indirectly causing rebuilds of numerous other crates as well.
|
||||
|
||||
The `solana-crate-features` offers a workaround to this "feature thrashing"
|
||||
problem by explicitly declaring all "C-like crates" with the union of all features
|
||||
that any other crate in the tree (either explicitly or implicitly) enable. All
|
||||
crates in the Solana source tree should depend on `solana-crate-features`.
|
||||
|
||||
### Adding new dependent crates
|
||||
When unnecessary `cargo` rebuilds are observed, the first step is to figure what
|
||||
dependent crate is suffering from feature thrashing.
|
||||
|
||||
This information is not readily available from the stock `cargo` program, so use
|
||||
the following steps to produce a custom `cargo` program that will output the
|
||||
necessary feature information to stderr during a build:
|
||||
```bash
|
||||
$ git clone git@github.com:rust-lang/cargo.git -b rust-1.37.0
|
||||
$ cd cargo
|
||||
$ git apply 0001-Print-package-features.patch
|
||||
$ cargo build
|
||||
$ mv ~/.cargo/bin/cargo ~/.cargo/bin/cargo.org
|
||||
$ cp ./target/debug/cargo ~/.cargo/bin/cargo
|
||||
```
|
||||
|
||||
Rebuild with the custom `cargo` and search for indications of crates getting
|
||||
built with different features (repeated runs of `./scripts/cargo-install-all.sh`
|
||||
work great for this). When the problematic crate is identified, add it as a
|
||||
dependency of `solana-crate-features` with the union of all observed enabled
|
||||
features for that crate.
|
||||
|
||||
### Appendix
|
||||
This command will enable some additional cargo log output that can help debug
|
||||
dependency problems as well:
|
||||
```bash
|
||||
export CARGO_LOG=cargo::core::compiler::fingerprint=info
|
||||
```
|
||||
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -55,7 +55,7 @@ else
|
|||
if [[ -n $NDEBUG ]]; then
|
||||
maybe_release=--release
|
||||
fi
|
||||
declare manifest_path="--manifest-path=$SOLANA_ROOT/$crate/Cargo.toml"
|
||||
declare manifest_path="--manifest-path=$SOLANA_ROOT/Cargo.toml"
|
||||
printf "cargo run $manifest_path $maybe_release $maybe_package --bin %s %s -- " "$program" "$features"
|
||||
}
|
||||
fi
|
||||
|
|
|
|||
11
net/net.sh
11
net/net.sh
|
|
@ -125,7 +125,6 @@ internalNodesLamports=
|
|||
maybeNoSnapshot=""
|
||||
maybeSkipLedgerVerify=""
|
||||
maybeDisableAirdrops=""
|
||||
buildProfile="--release"
|
||||
debugBuild=false
|
||||
doBuild=true
|
||||
|
||||
|
|
@ -276,8 +275,6 @@ while getopts "h?T:t:o:f:rD:c:Fn:i:d" opt "${shortArgs[@]}"; do
|
|||
esac
|
||||
done
|
||||
|
||||
$debugBuild && buildProfile=""
|
||||
|
||||
loadConfigFile
|
||||
|
||||
if [[ -n $numFullnodesRequested ]]; then
|
||||
|
|
@ -332,9 +329,15 @@ build() {
|
|||
# shellcheck source=/dev/null
|
||||
source target/perf-libs/env.sh
|
||||
fi
|
||||
|
||||
buildVariant=
|
||||
if $debugBuild; then
|
||||
buildVariant=debug
|
||||
fi
|
||||
|
||||
$MAYBE_DOCKER bash -c "
|
||||
set -ex
|
||||
scripts/cargo-install-all.sh farf \"$cargoFeatures\" \"$buildProfile\"
|
||||
scripts/cargo-install-all.sh farf \"$cargoFeatures\" \"$buildVariant\"
|
||||
if [[ -n \"$customPrograms\" ]]; then
|
||||
scripts/cargo-install-custom-programs.sh farf $customPrograms
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -16,16 +16,18 @@ if [[ -z $1 ]]; then
|
|||
fi
|
||||
|
||||
installDir="$(mkdir -p "$1"; cd "$1"; pwd)"
|
||||
cargo=cargo
|
||||
cargoFeatures="$2"
|
||||
debugBuild="$3"
|
||||
|
||||
buildProfile="$3"
|
||||
if [[ -n "$buildProfile" ]]; then
|
||||
binTargetDir="release"
|
||||
else
|
||||
binTargetDir="debug"
|
||||
buildVariant=release
|
||||
maybeReleaseFlag=--release
|
||||
if [[ -n "$debugBuild" ]]; then
|
||||
maybeReleaseFlag=
|
||||
buildVariant=debug
|
||||
fi
|
||||
|
||||
echo "Install location: $installDir"
|
||||
echo "Install location: $installDir ($buildVariant)"
|
||||
|
||||
cd "$(dirname "$0")"/..
|
||||
|
||||
|
|
@ -34,36 +36,45 @@ SECONDS=0
|
|||
(
|
||||
set -x
|
||||
# shellcheck disable=SC2086 # Don't want to double quote $rust_version
|
||||
cargo $rust_version build --all $buildProfile --features="$cargoFeatures"
|
||||
$cargo $rust_version build --all $maybeReleaseFlag --features="$cargoFeatures"
|
||||
)
|
||||
|
||||
BIN_CRATES=(
|
||||
drone
|
||||
gossip
|
||||
install
|
||||
keygen
|
||||
ledger-tool
|
||||
replicator
|
||||
validator
|
||||
cli
|
||||
bench-exchange
|
||||
bench-tps
|
||||
BINS=(
|
||||
solana-drone
|
||||
solana-gossip
|
||||
solana-install
|
||||
solana-install-init
|
||||
solana-keygen
|
||||
solana-ledger-tool
|
||||
solana-replicator
|
||||
solana-validator
|
||||
solana
|
||||
solana-bench-exchange
|
||||
solana-bench-tps
|
||||
)
|
||||
|
||||
#XXX: Ensure `solana-genesis` is built LAST!
|
||||
# See https://github.com/solana-labs/solana/issues/5826
|
||||
BIN_CRATES+=( "genesis" )
|
||||
BINS+=(solana-genesis)
|
||||
|
||||
for crate in "${BIN_CRATES[@]}"; do
|
||||
(
|
||||
set -x
|
||||
# shellcheck disable=SC2086 # Don't want to double quote $rust_version
|
||||
cargo $rust_version install --force --path "$crate" --root "$installDir" --features="$cargoFeatures"
|
||||
)
|
||||
binArgs=()
|
||||
for bin in "${BINS[@]}"; do
|
||||
binArgs+=(--bin "$bin")
|
||||
done
|
||||
|
||||
(
|
||||
set -x
|
||||
# shellcheck disable=SC2086 # Don't want to double quote $rust_version
|
||||
$cargo $rust_version build $maybeReleaseFlag "${binArgs[@]}" --features="$cargoFeatures"
|
||||
)
|
||||
|
||||
mkdir -p "$installDir/bin"
|
||||
for bin in "${BINS[@]}"; do
|
||||
cp -fv "target/$buildVariant/$bin" "$installDir"/bin
|
||||
done
|
||||
|
||||
for dir in programs/*; do
|
||||
for program in echo target/$binTargetDir/deps/libsolana_"$(basename "$dir")".{so,dylib,dll}; do
|
||||
for program in echo target/$buildVariant/deps/libsolana_"$(basename "$dir")".{so,dylib,dll}; do
|
||||
if [[ -f $program ]]; then
|
||||
mkdir -p "$installDir/bin/deps"
|
||||
rm -f "$installDir/bin/deps/$(basename "$program")"
|
||||
|
|
@ -72,5 +83,4 @@ for dir in programs/*; do
|
|||
done
|
||||
done
|
||||
|
||||
du -a "$installDir"
|
||||
echo "Done after $SECONDS seconds"
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ default = [
|
|||
"solana-ed25519-dalek",
|
||||
"solana-logger",
|
||||
"untrusted",
|
||||
"solana-crate-features"
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
|
|
@ -46,4 +47,5 @@ serde_json = { version = "1.0.40", optional = true }
|
|||
sha2 = "0.8.0"
|
||||
solana-ed25519-dalek = { version = "0.2.0", optional = true }
|
||||
solana-logger = { path = "../logger", version = "0.19.0-pre0", optional = true }
|
||||
untrusted = { version = "0.7.0", optional = true }
|
||||
untrusted = { version = "0.7.0", optional = true }
|
||||
solana-crate-features = { path = "../crate-features", version = "0.19.0-pre0", optional = true }
|
||||
|
|
|
|||
Loading…
Reference in New Issue