Auto merge of #4873 - str4d:rpc-test-blocks, r=str4d

Enable mininodes to construct blocks in RPC tests

Includes changes to disable difficulty adjustment on regtest, cherry-picked from:
- bitcoin/bitcoin#6853
- bitcoin/bitcoin#13046

The ZIP 143/243 logic is ported from https://github.com/zcash-hackworks/zcash-test-vectors

qa/rpc-tests/test_framework/blocktools.py

@@ -16,10 +16,12 @@ def create_block(hashprev, coinbase, nTime=None, nBits=None, hashFinalSaplingRoo
     else:
         block.nTime = nTime
     block.hashPrevBlock = hashprev
-    if hashFinalSaplingRoot is not None:
+    if hashFinalSaplingRoot is None:
-        block.hashFinalSaplingRoot = hashFinalSaplingRoot
+        # By default NUs up to Sapling are active from block 1, so we set this to the empty root.
+        hashFinalSaplingRoot = 0x3e49b5f954aa9d3545bc6c37744661eea48d7c34e3000d82b7f0010c30f4c2fb
+    block.hashFinalSaplingRoot = hashFinalSaplingRoot
     if nBits is None:
-        block.nBits = 0x200f0f0f # Will break after a difficulty adjustment...
+        block.nBits = 0x200f0f0f # difficulty retargeting is disabled in REGTEST chainparams
     else:
         block.nBits = nBits
     block.vtx.append(coinbase)

qa/rpc-tests/test_framework/script.py

@@ -19,9 +19,11 @@ if sys.version > '3':
     bchr = lambda x: bytes([x])
     bord = lambda x: x
 
+from pyblake2 import blake2b
 import struct
 
 from test_framework import bignum
+from test_framework.mininode import (CTransaction, CTxOut, hash256, ser_string, ser_uint256)
 
 MAX_SCRIPT_SIZE = 10000
 MAX_SCRIPT_ELEMENT_SIZE = 520
@@ -340,7 +342,6 @@ VALID_OPCODES = {
     OP_SHA256,
     OP_HASH160,
     OP_HASH256,
-    OP_CODESEPARATOR,
     OP_CHECKSIG,
     OP_CHECKSIGVERIFY,
     OP_CHECKMULTISIG,
@@ -814,3 +815,159 @@ class CScript(bytes):
                     n += 20
             lastOpcode = opcode
         return n
+
+
+SIGHASH_ALL = 1
+SIGHASH_NONE = 2
+SIGHASH_SINGLE = 3
+SIGHASH_ANYONECANPAY = 0x80
+
+def getHashPrevouts(tx):
+    digest = blake2b(digest_size=32, person=b'ZcashPrevoutHash')
+    for x in tx.vin:
+        digest.update(x.prevout.serialize())
+    return digest.digest()
+
+def getHashSequence(tx):
+    digest = blake2b(digest_size=32, person=b'ZcashSequencHash')
+    for x in tx.vin:
+        digest.update(struct.pack('<I', x.nSequence))
+    return digest.digest()
+
+def getHashOutputs(tx):
+    digest = blake2b(digest_size=32, person=b'ZcashOutputsHash')
+    for x in tx.vout:
+        digest.update(x.serialize())
+    return digest.digest()
+
+def getHashJoinSplits(tx):
+    digest = blake2b(digest_size=32, person=b'ZcashJSplitsHash')
+    for jsdesc in tx.vJoinSplit:
+        digest.update(jsdesc.serialize())
+    digest.update(tx.joinSplitPubKey)
+    return digest.digest()
+
+def getHashShieldedSpends(tx):
+    digest = blake2b(digest_size=32, person=b'ZcashSSpendsHash')
+    for desc in tx.shieldedSpends:
+        # We don't pass in serialized form of desc as spendAuthSig is not part of the hash
+        digest.update(ser_uint256(desc.cv))
+        digest.update(ser_uint256(desc.anchor))
+        digest.update(ser_uint256(desc.nullifier))
+        digest.update(ser_uint256(desc.rk))
+        digest.update(desc.proof)
+    return digest.digest()
+
+def getHashShieldedOutputs(tx):
+    digest = blake2b(digest_size=32, person=b'ZcashSOutputHash')
+    for desc in tx.shieldedOutputs:
+        digest.update(desc.serialize())
+    return digest.digest()
+
+
+def SignatureHash(script, txTo, inIdx, hashtype, amount, consensusBranchId):
+    """Consensus-correct SignatureHash"""
+    if inIdx >= len(txTo.vin):
+        raise ValueError("inIdx %d out of range (%d)" % (inIdx, len(txTo.vin)))
+
+    if consensusBranchId != 0:
+        # ZIP 243
+        hashPrevouts = b'\x00'*32
+        hashSequence = b'\x00'*32
+        hashOutputs = b'\x00'*32
+        hashJoinSplits = b'\x00'*32
+        hashShieldedSpends = b'\x00'*32
+        hashShieldedOutputs = b'\x00'*32
+
+        if not (hashtype & SIGHASH_ANYONECANPAY):
+            hashPrevouts = getHashPrevouts(txTo)
+
+        if (not (hashtype & SIGHASH_ANYONECANPAY)) and \
+            (hashtype & 0x1f) != SIGHASH_SINGLE and \
+            (hashtype & 0x1f) != SIGHASH_NONE:
+            hashSequence = getHashSequence(txTo)
+
+        if (hashtype & 0x1f) != SIGHASH_SINGLE and \
+            (hashtype & 0x1f) != SIGHASH_NONE:
+            hashOutputs = getHashOutputs(txTo)
+        elif (hashtype & 0x1f) == SIGHASH_SINGLE and \
+            0 <= inIdx and inIdx < len(txTo.vout):
+            digest = blake2b(digest_size=32, person=b'ZcashOutputsHash')
+            digest.update(txTo.vout[inIdx].serialize())
+            hashOutputs = digest.digest()
+
+        if len(txTo.vJoinSplit) > 0:
+            hashJoinSplits = getHashJoinSplits(txTo)
+
+        if len(txTo.shieldedSpends) > 0:
+            hashShieldedSpends = getHashShieldedSpends(txTo)
+
+        if len(txTo.shieldedOutputs) > 0:
+            hashShieldedOutputs = getHashShieldedOutputs(txTo)
+
+        digest = blake2b(
+            digest_size=32,
+            person=b'ZcashSigHash' + struct.pack('<I', consensusBranchId),
+        )
+
+        digest.update(struct.pack('<I', (int(txTo.fOverwintered)<<31) | txTo.nVersion))
+        digest.update(struct.pack('<I', txTo.nVersionGroupId))
+        digest.update(hashPrevouts)
+        digest.update(hashSequence)
+        digest.update(hashOutputs)
+        digest.update(hashJoinSplits)
+        digest.update(hashShieldedSpends)
+        digest.update(hashShieldedOutputs)
+        digest.update(struct.pack('<I', txTo.nLockTime))
+        digest.update(struct.pack('<I', txTo.nExpiryHeight))
+        digest.update(struct.pack('<Q', txTo.valueBalance))
+        digest.update(struct.pack('<I', hashtype))
+
+        if inIdx is not None:
+            digest.update(txTo.vin[inIdx].prevout.serialize())
+            digest.update(ser_string(script))
+            digest.update(struct.pack('<Q', amount))
+            digest.update(struct.pack('<I', txTo.vin[inIdx].nSequence))
+
+        return (digest.digest(), None)
+    else:
+        # Pre-Overwinter
+        txtmp = CTransaction(txTo)
+
+        for txin in txtmp.vin:
+            txin.scriptSig = b''
+        txtmp.vin[inIdx].scriptSig = script
+
+        if (hashtype & 0x1f) == SIGHASH_NONE:
+            txtmp.vout = []
+
+            for i in range(len(txtmp.vin)):
+                if i != inIdx:
+                    txtmp.vin[i].nSequence = 0
+
+        elif (hashtype & 0x1f) == SIGHASH_SINGLE:
+            outIdx = inIdx
+            if outIdx >= len(txtmp.vout):
+                raise ValueError("outIdx %d out of range (%d)" % (outIdx, len(txtmp.vout)))
+
+            tmp = txtmp.vout[outIdx]
+            txtmp.vout = []
+            for i in range(outIdx):
+                txtmp.vout.append(CTxOut())
+            txtmp.vout.append(tmp)
+
+            for i in range(len(txtmp.vin)):
+                if i != inIdx:
+                    txtmp.vin[i].nSequence = 0
+
+        if hashtype & SIGHASH_ANYONECANPAY:
+            tmp = txtmp.vin[inIdx]
+            txtmp.vin = []
+            txtmp.vin.append(tmp)
+
+        s = txtmp.serialize()
+        s += struct.pack(b"<I", hashtype)
+
+        hash = hash256(s)
+
+        return (hash, None)

src/chainparams.cpp

@@ -104,6 +104,7 @@ public:
         consensus.nPreBlossomPowTargetSpacing = Consensus::PRE_BLOSSOM_POW_TARGET_SPACING;
         consensus.nPostBlossomPowTargetSpacing = Consensus::POST_BLOSSOM_POW_TARGET_SPACING;
         consensus.nPowAllowMinDifficultyBlocksAfterHeight = boost::none;
+        consensus.fPowNoRetargeting = false;
         consensus.vUpgrades[Consensus::BASE_SPROUT].nProtocolVersion = 170002;
         consensus.vUpgrades[Consensus::BASE_SPROUT].nActivationHeight =
             Consensus::NetworkUpgrade::ALWAYS_ACTIVE;
@@ -381,6 +382,7 @@ public:
         consensus.nPreBlossomPowTargetSpacing = Consensus::PRE_BLOSSOM_POW_TARGET_SPACING;
         consensus.nPostBlossomPowTargetSpacing = Consensus::POST_BLOSSOM_POW_TARGET_SPACING;
         consensus.nPowAllowMinDifficultyBlocksAfterHeight = 299187;
+        consensus.fPowNoRetargeting = false;
         consensus.vUpgrades[Consensus::BASE_SPROUT].nProtocolVersion = 170002;
         consensus.vUpgrades[Consensus::BASE_SPROUT].nActivationHeight =
             Consensus::NetworkUpgrade::ALWAYS_ACTIVE;
@@ -624,6 +626,7 @@ public:
         consensus.nPreBlossomPowTargetSpacing = Consensus::PRE_BLOSSOM_POW_TARGET_SPACING;
         consensus.nPostBlossomPowTargetSpacing = Consensus::POST_BLOSSOM_POW_TARGET_SPACING;
         consensus.nPowAllowMinDifficultyBlocksAfterHeight = 0;
+        consensus.fPowNoRetargeting = true;
         consensus.vUpgrades[Consensus::BASE_SPROUT].nProtocolVersion = 170002;
         consensus.vUpgrades[Consensus::BASE_SPROUT].nActivationHeight =
             Consensus::NetworkUpgrade::ALWAYS_ACTIVE;
@@ -719,11 +722,16 @@ public:
         consensus.vFundingStreams[idx] = fs;
     }
 
-    void UpdateRegtestPow(int64_t nPowMaxAdjustDown, int64_t nPowMaxAdjustUp, uint256 powLimit)
+    void UpdateRegtestPow(
+        int64_t nPowMaxAdjustDown,
+        int64_t nPowMaxAdjustUp,
+        uint256 powLimit,
+        bool noRetargeting)
     {
         consensus.nPowMaxAdjustDown = nPowMaxAdjustDown;
         consensus.nPowMaxAdjustUp = nPowMaxAdjustUp;
         consensus.powLimit = powLimit;
+        consensus.fPowNoRetargeting = noRetargeting;
     }
 
     void SetRegTestZIP209Enabled() {
@@ -816,6 +824,11 @@ void UpdateFundingStreamParameters(Consensus::FundingStreamIndex idx, Consensus:
     regTestParams.UpdateFundingStreamParameters(idx, fs);
 }
 
-void UpdateRegtestPow(int64_t nPowMaxAdjustDown, int64_t nPowMaxAdjustUp, uint256 powLimit) {
+void UpdateRegtestPow(
-    regTestParams.UpdateRegtestPow(nPowMaxAdjustDown, nPowMaxAdjustUp, powLimit);
+    int64_t nPowMaxAdjustDown,
+    int64_t nPowMaxAdjustUp,
+    uint256 powLimit,
+    bool noRetargeting)
+{
+    regTestParams.UpdateRegtestPow(nPowMaxAdjustDown, nPowMaxAdjustUp, powLimit, noRetargeting);
 }

src/chainparams.h

@@ -144,7 +144,11 @@ void SelectParams(const std::string& chain);
  */
 void UpdateNetworkUpgradeParameters(Consensus::UpgradeIndex idx, int nActivationHeight);
 
-void UpdateRegtestPow(int64_t nPowMaxAdjustDown, int64_t nPowMaxAdjustUp, uint256 powLimit);
+void UpdateRegtestPow(
+    int64_t nPowMaxAdjustDown,
+    int64_t nPowMaxAdjustUp,
+    uint256 powLimit,
+    bool noRetargeting);
 
 /**
  * Allows modifying the regtest funding stream parameters.

src/consensus/params.h

@@ -262,6 +262,7 @@ struct Params {
     unsigned int nEquihashK = 0;
     uint256 powLimit;
     boost::optional<uint32_t> nPowAllowMinDifficultyBlocksAfterHeight;
+    bool fPowNoRetargeting;
     int64_t nPowAveragingWindow;
     int64_t nPowMaxAdjustDown;
     int64_t nPowMaxAdjustUp;

src/pow.cpp

@@ -23,6 +23,10 @@ unsigned int GetNextWorkRequired(const CBlockIndex* pindexLast, const CBlockHead
     if (pindexLast == NULL)
         return nProofOfWorkLimit;
 
+    // Regtest
+    if (params.fPowNoRetargeting)
+        return pindexLast->nBits;
+
     {
         // Comparing to pindexLast->nHeight with >= because this function
         // returns the work required for the block after pindexLast.

src/utiltest.cpp

@@ -212,13 +212,13 @@ const Consensus::Params& RegtestActivateBlossom(bool updatePow, int blossomActiv
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, Consensus::NetworkUpgrade::ALWAYS_ACTIVE);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_BLOSSOM, blossomActivationHeight);
     if (updatePow) {
-        UpdateRegtestPow(32, 16, uint256S("0007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"));
+        UpdateRegtestPow(32, 16, uint256S("0007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), false);
     }
     return Params().GetConsensus();
 }
 
 void RegtestDeactivateBlossom() {
-    UpdateRegtestPow(0, 0, uint256S("0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f"));
+    UpdateRegtestPow(0, 0, uint256S("0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f"), true);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_BLOSSOM, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
@@ -232,13 +232,13 @@ const Consensus::Params& RegtestActivateHeartwood(bool updatePow, int heartwoodA
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_BLOSSOM, Consensus::NetworkUpgrade::ALWAYS_ACTIVE);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_HEARTWOOD, heartwoodActivationHeight);
     if (updatePow) {
-        UpdateRegtestPow(32, 16, uint256S("0007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"));
+        UpdateRegtestPow(32, 16, uint256S("0007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), false);
     }
     return Params().GetConsensus();
 }
 
 void RegtestDeactivateHeartwood() {
-    UpdateRegtestPow(0, 0, uint256S("0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f"));
+    UpdateRegtestPow(0, 0, uint256S("0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f"), true);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_HEARTWOOD, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_BLOSSOM, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
@@ -254,7 +254,7 @@ const Consensus::Params& RegtestActivateCanopy(bool updatePow, int canopyActivat
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_HEARTWOOD, Consensus::NetworkUpgrade::ALWAYS_ACTIVE);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_CANOPY, canopyActivationHeight);
     if (updatePow) {
-        UpdateRegtestPow(32, 16, uint256S("0007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"));
+        UpdateRegtestPow(32, 16, uint256S("0007ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff"), false);
     }
     return Params().GetConsensus();
 }
@@ -264,7 +264,7 @@ const Consensus::Params& RegtestActivateCanopy() {
 }
 
 void RegtestDeactivateCanopy() {
-    UpdateRegtestPow(0, 0, uint256S("0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f"));
+    UpdateRegtestPow(0, 0, uint256S("0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f"), true);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_CANOPY, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_HEARTWOOD, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);
     UpdateNetworkUpgradeParameters(Consensus::UPGRADE_BLOSSOM, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT);