Backpropagate Cargo.lock updates to all lock files (#9180)

* Experiment to backpropagate Cargo.lock updates to all lock files

* Move most of dependabot-specific code to its own file

* Various cleanups

* Fine tune..

* Clean up shells and stop obscure API...

ci/buildkite.yml

@@ -5,6 +5,13 @@
 # Release tags use buildkite-release.yml instead
 
 steps:
+  - command: "ci/dependabot-pr.sh"
+    name: "dependabot"
+    timeout_in_minutes: 5
+    if: build.env("GITHUB_USER") == "dependabot-preview[bot]"
+
+  - wait
+
   - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh"
     name: "checks"
     timeout_in_minutes: 20

ci/dependabot-pr.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd "$(dirname "$0")/.."
+
+if ! echo "$BUILDKITE_BRANCH" | grep -E '^pull/[0-9]+/head$'; then
+  echo "not pull request!?" >&2
+  exit 1
+fi
+
+source ci/rust-version.sh stable
+
+ci/docker-run.sh $rust_nightly_docker_image ci/dependabot-updater.sh
+
+if [[ $(git status --short :**/Cargo.lock | wc -l) -eq 0 ]]; then
+  echo --- ok
+  exit 0
+fi
+
+echo --- "(FAILING) Backpropagating dependabot-triggered Cargo.lock updates"
+
+name="dependabot-buildkite"
+api_base="https://api.github.com/repos/solana-labs/solana/pulls"
+pr_num=$(echo "$BUILDKITE_BRANCH" | grep -Eo '[0-9]+')
+branch=$(curl -s "$api_base/$pr_num" | python -c 'import json,sys;print json.load(sys.stdin)["head"]["ref"]')
+
+git add :**/Cargo.lock
+EMAIL="dependabot-buildkite@noreply.solana.com" \
+  GIT_AUTHOR_NAME="$name" \
+  GIT_COMMITTER_NAME="$name" \
+  git commit -m "[auto-commit] Update all Cargo lock files"
+git push origin "HEAD:$branch"
+
+echo "Source branch is updated; failing this build for the next"
+exit 1

ci/dependabot-updater.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -ex
+cd "$(dirname "$0")/.."
+source ci/_
+
+commit_range="$(git merge-base HEAD origin/master)..HEAD"
+parsed_update_args="$(
+  git log "$commit_range" --author "dependabot-preview" --oneline -n1 |
+    grep -o 'Bump.*$' |
+    sed -r 's/Bump ([^ ]+) from [^ ]+ to ([^ ]+)/-p \1 --precise \2/'
+)"
+package=$(echo "$parsed_update_args" | awk '{print $2}')
+if [[ -n $parsed_update_args ]]; then
+  # shellcheck disable=SC2086
+  _ scripts/cargo-for-all-lock-files.sh \
+    "$(git grep --files-with-matches "$package" :**/Cargo.lock)" -- \
+    update $parsed_update_args
+fi
+
+echo --- ok

ci/test-checks.sh

@@ -13,6 +13,15 @@ export RUSTFLAGS="-D warnings"
 # Look for failed mergify.io backports
 _ git show HEAD --check --oneline
 
+if _ scripts/cargo-for-all-lock-files.sh check --locked; then
+  true
+else
+  check_status=$?
+  echo "Some Cargo.lock is outdated; please update them as well"
+  echo "protip: you can use ./scripts/cargo-for-all-lock-files.sh update ..."
+  exit "$check_status"
+fi
+
 _ cargo +"$rust_stable" fmt --all -- --check
 
 # Clippy gets stuck for unknown reasons if sdk-c is included in the build, so check it separately.

scripts/cargo-for-all-lock-files.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -e
+
+shifted_args=()
+while [[ -n $1 ]]; do
+  if [[ $1 = -- ]]; then
+    escape_marker=found
+    shift
+    break
+  else
+    shifted_args+=("$1")
+    shift
+  fi
+done
+
+# When "--" appear at the first and shifted_args is empty, consume it here
+# to unambiguously pass and use any other "--" for cargo
+if [[ -n $escape_marker && ${#shifted_args[@]} -gt 0 ]]; then
+  files="${shifted_args[*]}"
+  for file in $files; do
+    if [[ $file = "${file%Cargo.lock}" ]]; then
+      echo "$0: unrecognizable as Cargo.lock path (prepend \"--\"?): $file" >&2
+      exit 1
+    fi
+  done
+  shifted_args=()
+else
+  files="$(git ls-files :**/Cargo.lock)"
+fi
+
+for lock_file in $files; do
+  (
+    set -x
+    cd "$(dirname "$lock_file")"
+    cargo "${shifted_args[@]}" "$@"
+  )
+done