From c66c731adf766b5085eefe0008b7c83fe02f5162 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 12 May 2017 13:00:15 +1200 Subject: [PATCH] Add a benchmark for calling ConnectBlock on a block with many inputs Requires placing block-107134.tar.gz (containing the block, and a fake CoinsDB containing its inputs) into the base directory of the repository. This can be generated using qa/zcash/create_benchmark_archive.py (see the script for usage details). To facilitate generation of the fake CoinsDB, an additional field 'valueZat' has been added to 'getrawtransaction' containing the integer number of zatoshis instead of a decimal number of ZEC. Closes #2355. --- qa/zcash/create_benchmark_archive.py | 236 +++++++++++++++++++++++++++ qa/zcash/performance-measurements.sh | 44 ++++- src/rpcrawtransaction.cpp | 1 + src/txdb.cpp | 3 + src/txdb.h | 1 + src/wallet/rpcwallet.cpp | 5 + src/zcbenchmarks.cpp | 83 ++++++++++ src/zcbenchmarks.h | 1 + 8 files changed, 369 insertions(+), 5 deletions(-) create mode 100644 qa/zcash/create_benchmark_archive.py diff --git a/qa/zcash/create_benchmark_archive.py b/qa/zcash/create_benchmark_archive.py new file mode 100644 index 000000000..c61b10cf2 --- /dev/null +++ b/qa/zcash/create_benchmark_archive.py @@ -0,0 +1,236 @@ +import binascii +import json +import plyvel +import progressbar +import os +import struct +import subprocess +import sys + +ZCASH_CLI = './src/zcash-cli' +USAGE = """ +Requirements: +- faketime +- tar +- %s (edit ZCASH_CLI in this script to alter the path) +- A running mainnet zcashd using the default datadir with -txindex=1 + +Example usage: + +make -C src/leveldb/ +virtualenv venv +. venv/bin/activate +pip install --global-option=build_ext --global-option="-L$(pwd)/src/leveldb/" --global-option="-I$(pwd)/src/leveldb/include/" plyvel +pip install progressbar2 +LD_LIBRARY_PATH=src/leveldb python qa/zcash/create_benchmark_archive.py +""" % ZCASH_CLI + +def check_deps(): + if subprocess.call(['which', 'faketime', 'tar', ZCASH_CLI], stdout=subprocess.PIPE): + print USAGE + sys.exit() + +def encode_varint(n): + v = bytearray() + l = 0 + while True: + v.append((n & 0x7F) | (0x80 if l else 0x00)) + if (n <= 0x7F): + break + n = (n >> 7) - 1 + l += 1 + return bytes(v)[::-1] + +def decode_varint(v): + n = 0 + for ch in range(len(v)): + n = (n << 7) | (ord(v[ch]) & 0x7F) + if (ord(v[ch]) & 0x80): + n += 1 + else: + return n + +def compress_amount(n): + if n == 0: + return 0 + e = 0 + while (((n % 10) == 0) and e < 9): + n /= 10 + e += 1 + if e < 9: + d = (n % 10) + assert(d >= 1 and d <= 9) + n /= 10 + return 1 + (n*9 + d - 1)*10 + e + else: + return 1 + (n - 1)*10 + 9 + +OP_DUP = 0x76 +OP_EQUAL = 0x87 +OP_EQUALVERIFY = 0x88 +OP_HASH160 = 0xa9 +OP_CHECKSIG = 0xac +def to_key_id(script): + if len(script) == 25 and \ + script[0] == OP_DUP and \ + script[1] == OP_HASH160 and \ + script[2] == 20 and \ + script[23] == OP_EQUALVERIFY and \ + script[24] == OP_CHECKSIG: + return script[3:23] + return bytes() + +def to_script_id(script): + if len(script) == 23 and \ + script[0] == OP_HASH160 and \ + script[1] == 20 and \ + script[22] == OP_EQUAL: + return script[2:22] + return bytes() + +def to_pubkey(script): + if len(script) == 35 and \ + script[0] == 33 and \ + script[34] == OP_CHECKSIG and \ + (script[1] == 0x02 or script[1] == 0x03): + return script[1:34] + if len(script) == 67 and \ + script[0] == 65 and \ + script[66] == OP_CHECKSIG and \ + script[1] == 0x04: + return script[1:66] # assuming is fully valid + return bytes() + +def compress_script(script): + result = bytearray() + + key_id = to_key_id(script) + if key_id: + result.append(0x00) + result.extend(key_id) + return bytes(result) + + script_id = to_script_id(script) + if script_id: + result.append(0x01) + result.extend(script_id) + return bytes(result) + + pubkey = to_pubkey(script) + if pubkey: + result.append(0x00) + result.extend(pubkey[1:33]) + if pubkey[0] == 0x02 or pubkey[0] == 0x03: + result[0] = pubkey[0] + return bytes(result) + elif pubkey[0] == 0x04: + result[0] = 0x04 | (pubkey[64] & 0x01) + return bytes(result) + + size = len(script) + 6 + result.append(encode_varint(size)) + result.extend(script) + return bytes(result) + +def create_benchmark_archive(blk_hash): + blk = json.loads(subprocess.check_output([ZCASH_CLI, 'getblock', blk_hash])) + print 'Height: %d' % blk['height'] + print 'Transactions: %d' % len(blk['tx']) + + os.mkdir('benchmark') + with open('benchmark/block-%d.dat' % blk['height'], 'wb') as f: + f.write(binascii.unhexlify(subprocess.check_output([ZCASH_CLI, 'getblock', blk_hash, 'false']).strip())) + + txs = [json.loads(subprocess.check_output([ZCASH_CLI, 'getrawtransaction', tx, '1']) + ) for tx in blk['tx']] + + js_txs = len([tx for tx in txs if len(tx['vjoinsplit']) > 0]) + if js_txs: + print 'Block contains %d JoinSplit-containing transactions' % js_txs + return + + inputs = [(x['txid'], x['vout']) for tx in txs for x in tx['vin'] if x.has_key('txid')] + print 'Total inputs: %d' % len(inputs) + + unique_inputs = {} + for i in sorted(inputs): + if unique_inputs.has_key(i[0]): + unique_inputs[i[0]].append(i[1]) + else: + unique_inputs[i[0]] = [i[1]] + print 'Unique input transactions: %d' % len(unique_inputs) + + db_path = 'benchmark/block-%d-inputs' % blk['height'] + db = plyvel.DB(db_path, create_if_missing=True) + wb = db.write_batch() + bar = progressbar.ProgressBar(redirect_stdout=True) + print 'Collecting input coins for block' + for tx in bar(unique_inputs.keys()): + rawtx = json.loads(subprocess.check_output([ZCASH_CLI, 'getrawtransaction', tx, '1'])) + + mask_size = 0 + mask_code = 0 + b = 0 + while 2+b*8 < len(rawtx['vout']): + zero = True + i = 0 + while i < 8 and 2+b*8+i < len(rawtx['vout']): + if 2+b*8+i in unique_inputs[tx]: + zero = False + i += 1 + if not zero: + mask_size = b + 1 + mask_code += 1 + b += 1 + + coinbase = len(rawtx['vin']) == 1 and 'coinbase' in rawtx['vin'][0] + first = len(rawtx['vout']) > 0 and 0 in unique_inputs[tx] + second = len(rawtx['vout']) > 1 and 1 in unique_inputs[tx] + code = 8*(mask_code - (0 if first or second else 1)) + \ + (1 if coinbase else 0) + \ + (2 if first else 0) + \ + (4 if second else 0) + + coins = bytearray() + # Serialized format: + # - VARINT(nVersion) + coins.extend(encode_varint(rawtx['version'])) + # - VARINT(nCode) + coins.extend(encode_varint(code)) + # - unspentness bitvector, for vout[2] and further; least significant byte first + for b in range(mask_size): + avail = 0 + i = 0 + while i < 8 and 2+b*8+i < len(rawtx['vout']): + if 2+b*8+i in unique_inputs[tx]: + avail |= (1 << i) + i += 1 + coins.append(avail) + # - the non-spent CTxOuts (via CTxOutCompressor) + for i in range(len(rawtx['vout'])): + if i in unique_inputs[tx]: + coins.extend(encode_varint(compress_amount(int(rawtx['vout'][i]['valueZat'])))) + coins.extend(compress_script( + binascii.unhexlify(rawtx['vout'][i]['scriptPubKey']['hex']))) + # - VARINT(nHeight) + coins.extend(encode_varint(json.loads( + subprocess.check_output([ZCASH_CLI, 'getblockheader', rawtx['blockhash']]) + )['height'])) + + db_key = b'c' + bytes(binascii.unhexlify(tx)[::-1]) + db_val = bytes(coins) + wb.put(db_key, db_val) + + wb.write() + db.close() + + # Make reproducible archive + os.remove('%s/LOG' % db_path) + archive_name = 'block-%d.tar.gz' % blk['height'] + subprocess.check_call(['faketime', '2017-05-17T00:00:00Z', 'tar', 'czf', archive_name, '--mtime=2017-05-17T00:00:00Z', 'benchmark']) + print 'Created archive %s' % archive_name + subprocess.call(['rm', '-r', 'benchmark']) + +if __name__ == '__main__': + check_deps() + create_benchmark_archive('0000000007cdb809e48e51dd0b530e8f5073e0a9e9bd7ae920fe23e874658c74') diff --git a/qa/zcash/performance-measurements.sh b/qa/zcash/performance-measurements.sh index 2b6abee3e..6070c319d 100755 --- a/qa/zcash/performance-measurements.sh +++ b/qa/zcash/performance-measurements.sh @@ -1,8 +1,8 @@ #!/bin/bash -set -e - DATADIR=./benchmark-datadir +SHA256CMD="$(command -v sha256sum || echo shasum)" +SHA256ARGS="$(command -v sha256sum >/dev/null || echo '-a 256')" function zcash_rpc { ./src/zcash-cli -datadir="$DATADIR" -rpcwait -rpcuser=user -rpcpassword=password -rpcport=5983 "$@" @@ -24,7 +24,7 @@ function zcashd_generate { function zcashd_start { rm -rf "$DATADIR" - mkdir -p "$DATADIR" + mkdir -p "$DATADIR/regtest" touch "$DATADIR/zcash.conf" ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 & ZCASHD_PID=$! @@ -37,7 +37,7 @@ function zcashd_stop { function zcashd_massif_start { rm -rf "$DATADIR" - mkdir -p "$DATADIR" + mkdir -p "$DATADIR/regtest" touch "$DATADIR/zcash.conf" rm -f massif.out valgrind --tool=massif --time-unit=ms --massif-out-file=massif.out ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 & @@ -52,7 +52,7 @@ function zcashd_massif_stop { function zcashd_valgrind_start { rm -rf "$DATADIR" - mkdir -p "$DATADIR" + mkdir -p "$DATADIR/regtest" touch "$DATADIR/zcash.conf" rm -f valgrind.out valgrind --leak-check=yes -v --error-limit=no --log-file="valgrind.out" ./src/zcashd -regtest -datadir="$DATADIR" -rpcuser=user -rpcpassword=password -rpcport=5983 -showmetrics=0 & @@ -65,6 +65,28 @@ function zcashd_valgrind_stop { cat valgrind.out } +function extract_benchmark_data { + if [ -f "block-107134.tar.gz" ]; then + # Check the hash of the archive: + "$SHA256CMD" $SHA256ARGS -c < #include +#include #include #include #include @@ -9,6 +11,7 @@ #include "primitives/transaction.h" #include "base58.h" #include "crypto/equihash.h" +#include "chain.h" #include "chainparams.h" #include "consensus/validation.h" #include "main.h" @@ -17,6 +20,7 @@ #include "script/sign.h" #include "sodium.h" #include "streams.h" +#include "txdb.h" #include "utiltest.h" #include "wallet/wallet.h" @@ -322,3 +326,82 @@ double benchmark_increment_note_witnesses(size_t nTxs) return timer_stop(tv_start); } +// Fake the input of a given block +class FakeCoinsViewDB : public CCoinsViewDB { + uint256 hash; + ZCIncrementalMerkleTree t; + +public: + FakeCoinsViewDB(std::string dbName, uint256& hash) : CCoinsViewDB(dbName, 100, false, false), hash(hash) {} + + bool GetAnchorAt(const uint256 &rt, ZCIncrementalMerkleTree &tree) const { + if (rt == t.root()) { + tree = t; + return true; + } + return false; + } + + bool GetNullifier(const uint256 &nf) const { + return false; + } + + uint256 GetBestBlock() const { + return hash; + } + + uint256 GetBestAnchor() const { + return t.root(); + } + + bool BatchWrite(CCoinsMap &mapCoins, + const uint256 &hashBlock, + const uint256 &hashAnchor, + CAnchorsMap &mapAnchors, + CNullifiersMap &mapNullifiers) { + return false; + } + + bool GetStats(CCoinsStats &stats) const { + return false; + } +}; + +double benchmark_connectblock_slow() +{ + // Test for issue 2017-05-01.a + SelectParams(CBaseChainParams::MAIN); + CBlock block; + FILE* fp = fopen((GetDataDir() / "benchmark/block-107134.dat").string().c_str(), "rb"); + if (!fp) throw new std::runtime_error("Failed to open block data file"); + CAutoFile blkFile(fp, SER_DISK, CLIENT_VERSION); + blkFile >> block; + blkFile.fclose(); + + // Fake its inputs + auto hashPrev = uint256S("00000000159a41f468e22135942a567781c3f3dc7ad62257993eb3c69c3f95ef"); + FakeCoinsViewDB fakeDB("benchmark/block-107134-inputs", hashPrev); + CCoinsViewCache view(&fakeDB); + + // Fake the chain + CBlockIndex index(block); + index.nHeight = 107134; + CBlockIndex indexPrev; + indexPrev.phashBlock = &hashPrev; + indexPrev.nHeight = index.nHeight - 1; + index.pprev = &indexPrev; + mapBlockIndex.insert(std::make_pair(hashPrev, &indexPrev)); + + CValidationState state; + struct timeval tv_start; + timer_start(tv_start); + assert(ConnectBlock(block, state, &index, view, true)); + auto duration = timer_stop(tv_start); + + // Undo alterations to global state + mapBlockIndex.erase(hashPrev); + SelectParamsFromCommandLine(); + + return duration; +} + diff --git a/src/zcbenchmarks.h b/src/zcbenchmarks.h index b2bc2e373..b00b76789 100644 --- a/src/zcbenchmarks.h +++ b/src/zcbenchmarks.h @@ -15,5 +15,6 @@ extern double benchmark_verify_equihash(); extern double benchmark_large_tx(); extern double benchmark_try_decrypt_notes(size_t nAddrs); extern double benchmark_increment_note_witnesses(size_t nTxs); +extern double benchmark_connectblock_slow(); #endif