Merge #9880: Verify Tree-SHA512s in merge commits, enforce sigs are not SHA1

bbd7579 Fix regsig checking for subkey sigs in verify-commits (Matt Corallo)
d025bc7 Allow any subkey in verify-commits (Matt Corallo)
eddc77a Add comment re: why SHA1 is disabled (Peter Todd)
d9c450f Verify Tree-SHA512s in merge commits, enforce sigs are not SHA1 (Matt Corallo)
be908a6 Fail merge if there are any symlinks (Matt Corallo)

Tree-SHA512: bb66c59cc1c6b1c86d7d8be7adb0769c6598c0e28ad927409941f30af87d390521e82fc13700ee22e92db1bd571db3e19a152ec7b2c0349c6e06f5de62c0b65f

contrib/devtools/github-merge.py

@@ -70,6 +70,14 @@ def ask_prompt(text):
     print("",file=stderr)
     return reply
 
+def get_symlink_files():
+    files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', 'HEAD']).splitlines())
+    ret = []
+    for f in files:
+        if (int(f.decode('utf-8').split(" ")[0], 8) & 0o170000) == 0o120000:
+            ret.append(f.decode('utf-8').split("\t")[1])
+    return ret
+
 def tree_sha512sum():
     files = sorted(subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', '--name-only', 'HEAD']).splitlines())
     overall = hashlib.sha512()
@@ -200,6 +208,12 @@ def main():
             print("ERROR: Creating merge failed (already merged?).",file=stderr)
             exit(4)
 
+        symlink_files = get_symlink_files()
+        for f in symlink_files;
+            print("ERROR: File %s was a symlink" % f)
+        if len(symlink_files) > 0:
+            exit(4)
+
         # Put tree SHA512 into the message
         try:
             first_sha512 = tree_sha512sum()

contrib/verify-commits/gpg.sh

@@ -8,21 +8,31 @@ VALID=false
 REVSIG=false
 IFS='
 '
-for LINE in $(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null); do
+if [ "$BITCOIN_VERIFY_COMMITS_ALLOW_SHA1" = 1 ]; then
+	GPG_RES="$(echo "$INPUT" | gpg --trust-model always "$@" 2>/dev/null)"
+else
+	# Note how we've disabled SHA1 with the --weak-digest option, disabling
+	# signatures - including selfsigs - that use SHA1. While you might think that
+	# collision attacks shouldn't be an issue as they'd be an attack on yourself,
+	# in fact because what's being signed is a commit object that's
+	# semi-deterministically generated by untrusted input (the pull-req) in theory
+	# an attacker could construct a pull-req that results in a commit object that
+	# they've created a collision for. Not the most likely attack, but preventing
+	# it is pretty easy so we do so as a "belt-and-suspenders" measure.
+
+	GPG_RES="$(echo "$INPUT" | gpg --trust-model always --weak-digest sha1 "$@" 2>/dev/null)"
+fi
+for LINE in $(echo "$GPG_RES"); do
 	case "$LINE" in
 	"[GNUPG:] VALIDSIG "*)
 		while read KEY; do
-			case "$LINE" in "[GNUPG:] VALIDSIG $KEY "*) VALID=true;; esac
+			[ "${LINE#?GNUPG:? VALIDSIG * * * * * * * * * }" = "$KEY" ] && VALID=true
 		done < ./contrib/verify-commits/trusted-keys
 		;;
 	"[GNUPG:] REVKEYSIG "*)
 		[ "$BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG" != 1 ] && exit 1
-		while read KEY; do
-			case "$LINE" in "[GNUPG:] REVKEYSIG ${KEY#????????????????????????} "*)
-				REVSIG=true
-				GOODREVSIG="[GNUPG:] GOODSIG ${KEY#????????????????????????} "
-			esac
-		done < ./contrib/verify-commits/trusted-keys
+		REVSIG=true
+		GOODREVSIG="[GNUPG:] GOODSIG ${LINE#* * *}"
 		;;
 	esac
 done

contrib/verify-commits/trusted-keys

@@ -1,4 +1,4 @@
 71A3B16735405025D447E8F274810B012346C9A6
-3F1888C6DCA92A6499C4911FDBA1A67379A1A931
+133EAC179436F14A5CF1B794860FEB804E669320
 32EE5C4C3FA15CCADB46ABE529D4BCB6416F53EC
-FE09B823E6D83A3BC7983EAA2D7F2372E50FE137
+B8B3F1C0E58C15DB6A81D30C3648A882F4316B9B

contrib/verify-commits/trusted-sha512-root-commit

@@ -0,0 +1 @@
+b00ba6251f71fa1edaabdf809514e1bc3c67862e

contrib/verify-commits/verify-commits.sh

@@ -9,7 +9,10 @@
 DIR=$(dirname "$0")
 [ "/${DIR#/}" != "$DIR" ] && DIR=$(dirname "$(pwd)/$0")
 
+echo "Using verify-commits data from ${DIR}"
+
 VERIFIED_ROOT=$(cat "${DIR}/trusted-git-root")
+VERIFIED_SHA512_ROOT=$(cat "${DIR}/trusted-sha512-root-commit")
 REVSIG_ALLOWED=$(cat "${DIR}/allow-revsig-commits")
 
 HAVE_FAILED=false
@@ -17,18 +20,73 @@ IS_SIGNED () {
 	if [ $1 = $VERIFIED_ROOT ]; then
 		return 0;
 	fi
+
+	VERIFY_TREE=$2
+	NO_SHA1=$3
+	if [ $1 = $VERIFIED_SHA512_ROOT ]; then
+		if [ "$VERIFY_TREE" = "1" ]; then
+			echo "All Tree-SHA512s matched up to $VERIFIED_SHA512_ROOT" > /dev/stderr
+		fi
+		VERIFY_TREE=0
+		NO_SHA1=0
+	fi
+
+	if [ "$NO_SHA1" = "1" ]; then
+		export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=0
+	else
+		export BITCOIN_VERIFY_COMMITS_ALLOW_SHA1=1
+	fi
+
 	if [ "${REVSIG_ALLOWED#*$1}" != "$REVSIG_ALLOWED" ]; then
 		export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=1
 	else
 		export BITCOIN_VERIFY_COMMITS_ALLOW_REVSIG=0
 	fi
+
 	if ! git -c "gpg.program=${DIR}/gpg.sh" verify-commit $1 > /dev/null 2>&1; then
 		return 1;
 	fi
+
+	if [ "$VERIFY_TREE" = 1 ]; then
+		IFS_CACHE="$IFS"
+		IFS='
+'
+		for LINE in $(git ls-tree --full-tree -r $1); do
+			case "$LINE" in
+				"12"*)
+					echo "Repo contains symlinks" > /dev/stderr
+					IFS="$IFS_CACHE"
+					return 1
+					;;
+			esac
+		done
+		IFS="$IFS_CACHE"
+
+		FILE_HASHES=""
+		for FILE in $(git ls-tree --full-tree -r --name-only $1 | LANG=C sort); do
+			HASH=$(git cat-file blob $1:"$FILE" | sha512sum | { read FIRST OTHER; echo $FIRST; } )
+			[ "$FILE_HASHES" != "" ] && FILE_HASHES="$FILE_HASHES"$'\n'
+			FILE_HASHES="$FILE_HASHES$HASH  $FILE"
+		done
+		HASH_MATCHES=0
+		MSG="$(git show -s --format=format:%B $1 | tail -n1)"
+
+		case "$MSG  -" in
+			"Tree-SHA512: $(echo "$FILE_HASHES" | sha512sum)")
+				HASH_MATCHES=1;;
+		esac
+
+		if [ "$HASH_MATCHES" = "0" ]; then
+			echo "Tree-SHA512 did not match for commit $1" > /dev/stderr
+			HAVE_FAILED=true
+			return 1
+		fi
+	fi
+
 	local PARENTS
 	PARENTS=$(git show -s --format=format:%P $1)
 	for PARENT in $PARENTS; do
-		if IS_SIGNED $PARENT; then
+		if IS_SIGNED $PARENT $VERIFY_TREE $NO_SHA1; then
 			return 0;
 		fi
 		break
@@ -50,7 +108,13 @@ else
 	TEST_COMMIT="$1"
 fi
 
-IS_SIGNED "$TEST_COMMIT"
+DO_CHECKOUT_TEST=0
+if [ x"$2" = "x--tree-checks" ]; then
+	DO_CHECKOUT_TEST=1
+
+fi
+
+IS_SIGNED "$TEST_COMMIT" "$DO_CHECKOUT_TEST" 1
 RES=$?
 if [ "$RES" = 1 ]; then
 	if ! "$HAVE_FAILED"; then