Use python instead of slow shell script on verify-commits

.travis.yml

@@ -104,5 +104,5 @@ jobs:
         - test/lint/lint-all.sh
         - if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" -a "$TRAVIS_EVENT_TYPE" = "cron" ]; then
               while read LINE; do travis_retry gpg --keyserver hkp://subset.pool.sks-keyservers.net --recv-keys $LINE; done < contrib/verify-commits/trusted-keys &&
-              travis_wait 30 contrib/verify-commits/verify-commits.sh;
+              travis_wait 30 contrib/verify-commits/verify-commits.py;
           fi

contrib/verify-commits/README.md

@@ -7,18 +7,18 @@ are PGP signed (nearly always merge commits), as well as a script to verify
 commits against a trusted keys list.
 
 
-Using verify-commits.sh safely
+Using verify-commits.py safely
 ------------------------------
 
 Remember that you can't use an untrusted script to verify itself. This means
-that checking out code, then running `verify-commits.sh` against `HEAD` is
+that checking out code, then running `verify-commits.py` against `HEAD` is
-_not_ safe, because the version of `verify-commits.sh` that you just ran could
+_not_ safe, because the version of `verify-commits.py` that you just ran could
 be backdoored. Instead, you need to use a trusted version of verify-commits
 prior to checkout to make sure you're checking out only code signed by trusted
 keys:
 
     git fetch origin && \
-      ./contrib/verify-commits/verify-commits.sh origin/master && \
+      ./contrib/verify-commits/verify-commits.py origin/master && \
       git checkout origin/master
 
 Note that the above isn't a good UI/UX yet, and needs significant improvements
@@ -42,6 +42,6 @@ said key. In order to avoid bumping the root-of-trust `trusted-git-root`
 file, individual commits which were signed by such a key can be added to the
 `allow-revsig-commits` file. That way, the PGP signatures are still verified
 but no new commits can be signed by any expired/revoked key. To easily build a
-list of commits which need to be added, verify-commits.sh can be edited to test
+list of commits which need to be added, verify-commits.py can be edited to test
 each commit with BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG set to both 1 and 0, and
 those which need it set to 1 printed.

contrib/verify-commits/allow-incorrect-sha512-commits

@@ -0,0 +1,2 @@
+f8feaa4636260b599294c7285bcf1c8b7737f74e
+8040ae6fc576e9504186f2ae3ff2c8125de1095c

contrib/verify-commits/allow-unclean-merge-commits

@@ -0,0 +1,4 @@
+6052d509105790a26b3ad5df43dd61e7f1b24a12
+3798e5de334c3deb5f71302b782f6b8fbd5087f1
+326ffed09bfcc209a2efd6a2ebc69edf6bd200b5
+97d83739db0631be5d4ba86af3616014652c00ec

contrib/verify-commits/pre-push-hook.sh

@@ -12,9 +12,9 @@ while read LINE; do
     if [ "$4" != "refs/heads/master" ]; then
         continue
     fi
-    if ! ./contrib/verify-commits/verify-commits.sh $3 > /dev/null 2>&1; then
+    if ! ./contrib/verify-commits/verify-commits.py $3 > /dev/null 2>&1; then
         echo "ERROR: A commit is not signed, can't push"
-        ./contrib/verify-commits/verify-commits.sh
+        ./contrib/verify-commits/verify-commits.py
         exit 1
     fi
 done < /dev/stdin

contrib/verify-commits/verify-commits.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Verify commits against a trusted keys list."""
+import argparse
+import hashlib
+import os
+import subprocess
+import sys
+import time
+
+GIT = os.getenv('GIT', 'git')
+
+def tree_sha512sum(commit='HEAD'):
+    """Calculate the Tree-sha512 for the commit.
+
+    This is copied from github-merge.py."""
+
+    # request metadata for entire tree, recursively
+    files = []
+    blob_by_name = {}
+    for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines():
+        name_sep = line.index(b'\t')
+        metadata = line[:name_sep].split()  # perms, 'blob', blobid
+        assert metadata[1] == b'blob'
+        name = line[name_sep + 1:]
+        files.append(name)
+        blob_by_name[name] = metadata[2]
+
+    files.sort()
+    # open connection to git-cat-file in batch mode to request data for all blobs
+    # this is much faster than launching it per file
+    p = subprocess.Popen([GIT, 'cat-file', '--batch'], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+    overall = hashlib.sha512()
+    for f in files:
+        blob = blob_by_name[f]
+        # request blob
+        p.stdin.write(blob + b'\n')
+        p.stdin.flush()
+        # read header: blob, "blob", size
+        reply = p.stdout.readline().split()
+        assert reply[0] == blob and reply[1] == b'blob'
+        size = int(reply[2])
+        # hash the blob data
+        intern = hashlib.sha512()
+        ptr = 0
+        while ptr < size:
+            bs = min(65536, size - ptr)
+            piece = p.stdout.read(bs)
+            if len(piece) == bs:
+                intern.update(piece)
+            else:
+                raise IOError('Premature EOF reading git cat-file output')
+            ptr += bs
+        dig = intern.hexdigest()
+        assert p.stdout.read(1) == b'\n'  # ignore LF that follows blob data
+        # update overall hash with file hash
+        overall.update(dig.encode("utf-8"))
+        overall.update("  ".encode("utf-8"))
+        overall.update(f)
+        overall.update("\n".encode("utf-8"))
+    p.stdin.close()
+    if p.wait():
+        raise IOError('Non-zero return value executing git cat-file')
+    return overall.hexdigest()
+
+def main():
+    # Parse arguments
+    parser = argparse.ArgumentParser(usage='%(prog)s [options] [commit id]')
+    parser.add_argument('--disable-tree-check', action='store_false', dest='verify_tree', help='disable SHA-512 tree check')
+    parser.add_argument('--clean-merge', type=float, dest='clean_merge', default=float('inf'), help='Only check clean merge after <NUMBER> days ago (default: %(default)s)', metavar='NUMBER')
+    parser.add_argument('commit', nargs='?', default='HEAD', help='Check clean merge up to commit <commit>')
+    args = parser.parse_args()
+
+    # get directory of this program and read data files
+    dirname = os.path.dirname(os.path.abspath(__file__))
+    print("Using verify-commits data from " + dirname)
+    verified_root = open(dirname + "/trusted-git-root", "r").read().splitlines()[0]
+    verified_sha512_root = open(dirname + "/trusted-sha512-root-commit", "r").read().splitlines()[0]
+    revsig_allowed = open(dirname + "/allow-revsig-commits", "r").read().splitlines()
+    unclean_merge_allowed = open(dirname + "/allow-unclean-merge-commits", "r").read().splitlines()
+    incorrect_sha512_allowed = open(dirname + "/allow-incorrect-sha512-commits", "r").read().splitlines()
+
+    # Set commit and branch and set variables
+    current_commit = args.commit
+    if ' ' in current_commit:
+        print("Commit must not contain spaces", file=sys.stderr)
+        sys.exit(1)
+    verify_tree = args.verify_tree
+    no_sha1 = True
+    prev_commit = ""
+    initial_commit = current_commit
+    branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit], universal_newlines=True).splitlines()[0]
+
+    # Iterate through commits
+    while True:
+        if current_commit == verified_root:
+            print('There is a valid path from "{}" to {} where all commits are signed!'.format(initial_commit, verified_root))
+            sys.exit(0)
+        if current_commit == verified_sha512_root:
+            if verify_tree:
+                print("All Tree-SHA512s matched up to {}".format(verified_sha512_root), file=sys.stderr)
+            verify_tree = False
+            no_sha1 = False
+
+        os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_SHA1'] = "0" if no_sha1 else "1"
+        os.environ['BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG'] = "1" if current_commit in revsig_allowed else "0"
+
+        # Check that the commit (and parents) was signed with a trusted key
+        if subprocess.call([GIT, '-c', 'gpg.program={}/gpg.sh'.format(dirname), 'verify-commit', current_commit], stdout=subprocess.DEVNULL):
+            if prev_commit != "":
+                print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
+                print("Parents are:", file=sys.stderr)
+                parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], universal_newlines=True).splitlines()[0].split(' ')
+                for parent in parents:
+                    subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
+            else:
+                print("{} was not signed with a trusted key!".format(current_commit), file=sys.stderr)
+            sys.exit(1)
+
+        # Check the Tree-SHA512
+        if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
+            tree_hash = tree_sha512sum(current_commit)
+            if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], universal_newlines=True).splitlines():
+                print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
+                sys.exit(1)
+
+        # Merge commits should only have two parents
+        parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], universal_newlines=True).splitlines()[0].split(' ')
+        if len(parents) > 2:
+            print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
+            sys.exit(1)
+
+        # Check that the merge commit is clean
+        commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], universal_newlines=True).splitlines()[0])
+        check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60  # Only check commits in clean_merge days
+        allow_unclean = current_commit in unclean_merge_allowed
+        if len(parents) == 2 and check_merge and not allow_unclean:
+            current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], universal_newlines=True).splitlines()[0]
+            subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]])
+            subprocess.call([GIT, 'merge', '--no-ff', '--quiet', parents[1]], stdout=subprocess.DEVNULL)
+            recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD'], universal_newlines=True).splitlines()[0]
+            if current_tree != recreated_tree:
+                print("Merge commit {} is not clean".format(current_commit), file=sys.stderr)
+                subprocess.call([GIT, 'diff', current_commit])
+                subprocess.call([GIT, 'checkout', '--force', '--quiet', branch])
+                sys.exit(1)
+            subprocess.call([GIT, 'checkout', '--force', '--quiet', branch])
+
+        prev_commit = current_commit
+        current_commit = parents[0]
+
+if __name__ == '__main__':
+    main()

contrib/verify-commits/verify-commits.sh

@@ -1,153 +0,0 @@
-#!/bin/sh
-# Copyright (c) 2014-2016 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-DIR=$(dirname "$0")
-[ "/${DIR#/}" != "$DIR" ] && DIR=$(dirname "$(pwd)/$0")
-
-echo "Using verify-commits data from ${DIR}"
-
-VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root")
-VERIFIED_SHA512_ROOT=$(cat "${DIR}/trusted-sha512-root-commit")
-REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits")
-
-HAVE_GNU_SHA512=1
-[ ! -x "$(which sha512sum)" ] && HAVE_GNU_SHA512=0
-
-if [ x"$1" = "x" ]; then
-	CURRENT_COMMIT="HEAD"
-else
-	CURRENT_COMMIT="$1"
-fi
-
-if [ "${CURRENT_COMMIT#* }" != "$CURRENT_COMMIT" ]; then
-	echo "Commit must not contain spaces?" > /dev/stderr
-	exit 1
-fi
-
-VERIFY_TREE=0
-if [ x"$2" = "x--tree-checks" ]; then
-	VERIFY_TREE=1
-fi
-
-NO_SHA1=1
-PREV_COMMIT=""
-INITIAL_COMMIT="${CURRENT_COMMIT}"
-
-BRANCH="$(git rev-parse --abbrev-ref HEAD)"
-
-while true; do
-	if [ "$CURRENT_COMMIT" = $VERIFIED_ROOT ]; then
-		echo "There is a valid path from \"$INITIAL_COMMIT\" to $VERIFIED_ROOT where all commits are signed!"
-		exit 0
-	fi
-
-	if [ "$CURRENT_COMMIT" = $VERIFIED_SHA512_ROOT ]; then
-		if [ "$VERIFY_TREE" = "1" ]; then
-			echo "All Tree-SHA512s matched up to $VERIFIED_SHA512_ROOT" > /dev/stderr
-		fi
-		VERIFY_TREE=0
-		NO_SHA1=0
-	fi
-
-	if [ "$NO_SHA1" = "1" ]; then
-		export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=0
-	else
-		export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=1
-	fi
-
-	if [ "${REVSIG_ALLOWED#*$CURRENT_COMMIT}" != "$REVSIG_ALLOWED" ]; then
-		export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1
-	else
-		export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0
-	fi
-
-	if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit "$CURRENT_COMMIT" > /dev/null; then
-		if [ "$PREV_COMMIT" != "" ]; then
-			echo "No parent of $PREV_COMMIT was signed with a trusted key!" > /dev/stderr
-			echo "Parents are:" > /dev/stderr
-			PARENTS=$(git show -s --format=format:%P $PREV_COMMIT)
-			for PARENT in $PARENTS; do
-				git show -s $PARENT > /dev/stderr
-			done
-		else
-			echo "$CURRENT_COMMIT was not signed with a trusted key!" > /dev/stderr
-		fi
-		exit 1
-	fi
-
-	# We always verify the top of the tree
-	if [ "$VERIFY_TREE" = 1 -o "$PREV_COMMIT" = "" ]; then
-		IFS_CACHE="$IFS"
-		IFS='
-'
-		for LINE in $(git ls-tree --full-tree -r "$CURRENT_COMMIT"); do
-			case "$LINE" in
-				"12"*)
-					echo "Repo contains symlinks" > /dev/stderr
-					IFS="$IFS_CACHE"
-					exit 1
-					;;
-			esac
-		done
-		IFS="$IFS_CACHE"
-
-		FILE_HASHES=""
-		for FILE in $(git ls-tree --full-tree -r --name-only "$CURRENT_COMMIT" | LC_ALL=C sort); do
-			if [ "$HAVE_GNU_SHA512" = 1 ]; then
-				HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | sha512sum | { read FIRST _; echo $FIRST; } )
-			else
-				HASH=$(git cat-file blob "$CURRENT_COMMIT":"$FILE" | shasum -a 512 | { read FIRST _; echo $FIRST; } )
-			fi
-			[ "$FILE_HASHES" != "" ] && FILE_HASHES="$FILE_HASHES"'
-'
-			FILE_HASHES="$FILE_HASHES$HASH  $FILE"
-		done
-
-		if [ "$HAVE_GNU_SHA512" = 1 ]; then
-			TREE_HASH="$(echo "$FILE_HASHES" | sha512sum)"
-		else
-			TREE_HASH="$(echo "$FILE_HASHES" | shasum -a 512)"
-		fi
-		HASH_MATCHES=0
-		MSG="$(git show -s --format=format:%B "$CURRENT_COMMIT" | tail -n1)"
-
-		case "$MSG  -" in
-			"Tree-SHA512: $TREE_HASH")
-				HASH_MATCHES=1;;
-		esac
-
-		if [ "$HASH_MATCHES" = "0" ]; then
-			echo "Tree-SHA512 did not match for commit $CURRENT_COMMIT" > /dev/stderr
-			exit 1
-		fi
-	fi
-
-	PARENTS=$(git show -s --format=format:%P "$CURRENT_COMMIT")
-	PARENT1=${PARENTS%% *}
-	PARENT2=""
-	if [ "x$PARENT1" != "x$PARENTS" ]; then
-		PARENTX=${PARENTS#* }
-		PARENT2=${PARENTX%% *}
-		if [ "x$PARENT2" != "x$PARENTX" ]; then
-			echo "Commit $CURRENT_COMMIT is an octopus merge" > /dev/stderr
-			exit 1
-		fi
-	fi
-	if [ "x$PARENT2" != "x" ]; then
-		CURRENT_TREE="$(git show --format="%T" "$CURRENT_COMMIT")"
-		git checkout --force --quiet "$PARENT1"
-		git merge --no-ff --quiet "$PARENT2" >/dev/null
-		RECREATED_TREE="$(git show --format="%T" HEAD)"
-		if [ "$CURRENT_TREE" != "$RECREATED_TREE" ]; then
-			echo "Merge commit $CURRENT_COMMIT is not clean" > /dev/stderr
-			git diff "$CURRENT_COMMIT"
-			git checkout --force --quiet "$BRANCH"
-			exit 1
-		fi
-		git checkout --force --quiet "$BRANCH"
-	fi
-	PREV_COMMIT="$CURRENT_COMMIT"
-	CURRENT_COMMIT="$PARENT1"
-done