Auto merge of #4827 - str4d:commit-script-check, r=str4d

Verifier for scriptable changes

Includes changes from the following upstream PRs:
- bitcoin/bitcoin#10189
  - Excluding the `CNode` scripted changes.
- bitcoin/bitcoin#10480
- bitcoin/bitcoin#11390
- bitcoin/bitcoin#13281
  - Only the lint scripts we already have.
- bitcoin/bitcoin#13454
  - Only changes to scripts we already have.
- bitcoin/bitcoin#14864
- bitcoin/bitcoin#16327
  - Includes some portability fixes to other shell scripts.
- bitcoin/bitcoin#20069

autogen.sh

@@ -5,10 +5,11 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or https://www.opensource.org/licenses/mit-license.php .
 
+export LC_ALL=C
 set -e
 srcdir="$(dirname $0)"
 cd "$srcdir"
-if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="`which glibtoolize 2>/dev/null`"; then
+if [ -z ${LIBTOOLIZE} ] && GLIBTOOLIZE="$(command -v glibtoolize)"; then
   LIBTOOLIZE="${GLIBTOOLIZE}"
   export LIBTOOLIZE
 fi

contrib/devtools/gen-manpages.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 
+export LC_ALL=C
 TOPDIR=${TOPDIR:-$(git rev-parse --show-toplevel)}
 SRCDIR=${SRCDIR:-$TOPDIR/src}
 MANDIR=${MANDIR:-$TOPDIR/doc/man}
@@ -11,9 +12,9 @@ ZCASHTX=${ZCASHTX:-$SRCDIR/zcash-tx}
 [ ! -x $ZCASHD ] && echo "$ZCASHD not found or not executable." && exit 1
 
 # The autodetected version git tag can screw up manpage output a little bit
-ZECVERSTR=$($ZCASHCLI --version | head -n1 | awk '{ print $NF }')
-ZECVER=$(echo $ZECVERSTR | awk -F- '{ OFS="-"; NF--; print $0; }')
-ZECCOMMIT=$(echo $ZECVERSTR | awk -F- '{ print $NF }')
+read -r -a ZECVERSTR <<< "$($ZCASHCLI --version | head -n1 | awk '{ print $NF }')"
+read -r -a ZECVER <<< "$(echo $ZECVERSTR | awk -F- '{ OFS="-"; NF--; print $0; }')"
+read -r -a ZECCOMMIT <<< "$(echo $ZECVERSTR | awk -F- '{ print $NF }')"
 
 # Create a footer file with copyright content.
 # This gets autodetected fine for zcashd if --version-string is not set,

contrib/qos/tc.sh

@@ -1,3 +1,8 @@
+# Copyright (c) 2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://www.opensource.org/licenses/mit-license.php .
+
+export LC_ALL=C
 #network interface on which to limit traffic
 IF="eth0"
 #limit of the network interface in question

doc/developer-notes.md

@@ -243,4 +243,25 @@ ACK -  A loose ACK can be confusing. It's best to avoid them unless it's a docum
 
 NACK - Disagree with the code changes/concept. Should be accompanied by an explanation.
 
-See the [Development Guidelines](https://zcash.readthedocs.io/en/latest/rtd_pages/development_guidelines.html) documentation for preferred workflows, information on continuous integration and release versioning.
+See the [Development Guidelines](https://zcash.readthedocs.io/en/latest/rtd_pages/development_guidelines.html) documentation for preferred workflows, information on continuous integration and release versioning.
+
+Scripted diffs
+--------------
+
+For reformatting and refactoring commits where the changes can be easily automated using a bash script, we use
+scripted-diff commits. The bash script is included in the commit message and our Travis CI job checks that
+the result of the script is identical to the commit. This aids reviewers since they can verify that the script
+does exactly what it's supposed to do. It is also helpful for rebasing (since the same script can just be re-run
+on the new master commit).
+
+To create a scripted-diff:
+
+- start the commit message with `scripted-diff:` (and then a description of the diff on the same line)
+- in the commit message include the bash script between lines containing just the following text:
+
+    - `-BEGIN VERIFY SCRIPT-`
+    - `-END VERIFY SCRIPT-`
+
+The scripted-diff is verified by the tool `test/lint/commit-script-check.sh`
+
+Commit `ccd074a5` is an example of a scripted-diff.

share/genbuild.sh

@@ -5,6 +5,7 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or https://www.opensource.org/licenses/mit-license.php .
 
+export LC_ALL=C
 if [ $# -gt 1 ]; then
     cd "$2"
 fi
@@ -24,7 +25,7 @@ SUFFIX=""
 LAST_COMMIT_DATE=""
 if [ -e "$(which git 2>/dev/null)" -a "$(git rev-parse --is-inside-work-tree 2>/dev/null)" = "true" ]; then
     # clean 'dirty' status of touched files that haven't been modified
-    git diff >/dev/null 2>/dev/null 
+    git diff >/dev/null 2>/dev/null
 
     # if latest commit is tagged and not dirty, then override using the tag name
     RAWDESC=$(git describe --abbrev=0 2>/dev/null)

test/lint/README.md

@@ -0,0 +1,21 @@
+This folder contains lint scripts.
+
+commit-script-check.sh
+======================
+Verification of [scripted diffs](/doc/developer-notes.md#scripted-diffs).
+
+git-subtree-check.sh
+====================
+Run this script from the root of the repository to verify that a subtree matches the contents of
+the commit it claims to have been updated to.
+
+To use, make sure that you have fetched the upstream repository branch in which the subtree is
+maintained:
+* for `src/secp256k1`: https://github.com/bitcoin-core/secp256k1.git (branch master)
+* for `src/leveldb`: https://github.com/bitcoin-core/leveldb.git (branch bitcoin-fork)
+* for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master)
+* for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master)
+
+Usage: `git-subtree-check.sh DIR (COMMIT)`
+
+`COMMIT` may be omitted, in which case `HEAD` is used.

test/lint/commit-script-check.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+# Copyright (c) 2017 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or https://www.opensource.org/licenses/mit-license.php .
+
+# This simple script checks for commits beginning with: scripted-diff:
+# If found, looks for a script between the lines -BEGIN VERIFY SCRIPT- and
+# -END VERIFY SCRIPT-. If no ending is found, it reads until the end of the
+# commit message.
+
+# The resulting script should exactly transform the previous commit into the current
+# one. Any remaining diff signals an error.
+
+export LC_ALL=C
+if test "x$1" = "x"; then
+    echo "Usage: $0 <commit>..."
+    exit 1
+fi
+
+# Check that the working tree is clean (as this script performs hard resets).
+TREE_CHANGES=$(git diff HEAD | wc -l)
+if test "x$TREE_CHANGES" != "x0"; then
+    echo "The working tree is not clean."
+    exit 1
+fi
+
+RET=0
+PREV_BRANCH=$(git name-rev --name-only HEAD)
+PREV_HEAD=$(git rev-parse HEAD)
+for commit in $(git rev-list --reverse $1); do
+    if git rev-list -n 1 --pretty="%s" $commit | grep -q "^scripted-diff:"; then
+        git checkout --quiet $commit^ || exit
+        SCRIPT="$(git rev-list --format=%b -n1 $commit | sed '/^-BEGIN VERIFY SCRIPT-$/,/^-END VERIFY SCRIPT-$/{//!b};d')"
+        if test "x$SCRIPT" = "x"; then
+            echo "Error: missing script for: $commit"
+            echo "Failed"
+            RET=1
+        else
+            echo "Running script for: $commit"
+            echo "$SCRIPT"
+            (eval "$SCRIPT")
+            git --no-pager diff --exit-code $commit && echo "OK" || (echo "Failed"; false) || RET=1
+        fi
+        git reset --quiet --hard HEAD
+     else
+        if git rev-list "--format=%b" -n1 $commit | grep -q '^-\(BEGIN\|END\)[ a-zA-Z]*-$'; then
+            echo "Error: script block marker but no scripted-diff in title of commit $commit"
+            echo "Failed"
+            RET=1
+        fi
+    fi
+done
+git checkout --quiet $PREV_BRANCH 2>/dev/null || git checkout --quiet $PREV_HEAD
+exit $RET

test/lint/git-subtree-check.sh

@@ -1,5 +1,6 @@
 #!/bin/sh
 
+export LC_ALL=C
 DIR="$1"
 COMMIT="$2"
 if [ -z "$COMMIT" ]; then