From 31436e4372c8222c9c47fd1ec45a27407df2ede7 Mon Sep 17 00:00:00 2001 From: syd Date: Mon, 27 Nov 2017 22:24:27 -0500 Subject: [PATCH 001/725] Add assert_raises_message to the python test framework. --- qa/rpc-tests/test_framework/authproxy.py | 2 +- qa/rpc-tests/test_framework/util.py | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index 28b33e286..220083d78 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -55,7 +55,7 @@ log = logging.getLogger("BitcoinRPC") class JSONRPCException(Exception): def __init__(self, rpc_error): - Exception.__init__(self) + Exception.__init__(self, rpc_error.get("message")) self.error = rpc_error diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 8ac09f472..0c2fb1da7 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -360,11 +360,20 @@ def assert_greater_than(thing1, thing2): raise AssertionError("%s <= %s"%(str(thing1),str(thing2))) def assert_raises(exc, fun, *args, **kwds): + assert_raises_message(exc, None, fun, *args, **kwds) + +def assert_raises_message(ExceptionType, errstr, func, *args, **kwargs): + """ + Asserts that func throws and that the exception contains 'errstr' + in its message. + """ try: - fun(*args, **kwds) - except exc: - pass + func(*args, **kwargs) + except ExceptionType as e: + if errstr is not None and errstr not in str(e): + raise AssertionError("Invalid exception string: Couldn't find %r in %r" % ( + errstr, str(e))) except Exception as e: - raise AssertionError("Unexpected exception raised: "+type(e).__name__) + raise AssertionError("Unexpected exception raised: " + type(e).__name__) else: raise AssertionError("No exception raised") From 0bab947bc5e949655fa9371f81089fa72fc64b44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Sat, 6 Jan 2018 09:49:57 +0100 Subject: [PATCH 002/725] Get rid of implicit hidden dependencies between test .cpp files --- src/Makefile.test.include | 2 + src/test/base58_tests.cpp | 3 +- src/test/rpc_tests.cpp | 39 +----------- src/test/rpc_wallet_tests.cpp | 6 +- src/test/script_tests.cpp | 17 +---- src/test/sighash_tests.cpp | 3 +- src/test/test_util.cpp | 109 +++++++++++++++++++++++++++++++++ src/test/test_util.h | 15 +++++ src/test/transaction_tests.cpp | 55 +---------------- 9 files changed, 133 insertions(+), 116 deletions(-) create mode 100644 src/test/test_util.cpp create mode 100644 src/test/test_util.h diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 582ecb577..7c90e8754 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -83,6 +83,8 @@ BITCOIN_TESTS =\ test/skiplist_tests.cpp \ test/test_bitcoin.cpp \ test/test_bitcoin.h \ + test/test_util.cpp \ + test/test_util.h \ test/timedata_tests.cpp \ test/torcontrol_tests.cpp \ test/transaction_tests.cpp \ diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index 361d8b8ea..ec159ed28 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -14,14 +14,13 @@ #include "util.h" #include "utilstrencodings.h" #include "test/test_bitcoin.h" +#include "test/test_util.h" #include #include #include -extern UniValue read_json(const std::string& jsondata); - BOOST_FIXTURE_TEST_SUITE(base58_tests, BasicTestingSetup) // Goal: test low-level base58 encoding functionality diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index c2b29ab67..ebfca760a 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -9,51 +9,14 @@ #include "netbase.h" #include "test/test_bitcoin.h" +#include "test/test_util.h" -#include #include #include using namespace std; -UniValue -createArgs(int nRequired, const char* address1=NULL, const char* address2=NULL) -{ - UniValue result(UniValue::VARR); - result.push_back(nRequired); - UniValue addresses(UniValue::VARR); - if (address1) addresses.push_back(address1); - if (address2) addresses.push_back(address2); - result.push_back(addresses); - return result; -} - -UniValue CallRPC(string args) -{ - vector vArgs; - boost::split(vArgs, args, boost::is_any_of(" \t")); - string strMethod = vArgs[0]; - vArgs.erase(vArgs.begin()); - // Handle empty strings the same way as CLI - for (auto i = 0; i < vArgs.size(); i++) { - if (vArgs[i] == "\"\"") { - vArgs[i] = ""; - } - } - UniValue params = RPCConvertValues(strMethod, vArgs); - - rpcfn_type method = tableRPC[strMethod]->actor; - try { - UniValue result = (*method)(params, false); - return result; - } - catch (const UniValue& objError) { - throw runtime_error(find_value(objError, "message").get_str()); - } -} - - BOOST_FIXTURE_TEST_SUITE(rpc_tests, TestingSetup) BOOST_AUTO_TEST_CASE(rpc_rawparams) diff --git a/src/test/rpc_wallet_tests.cpp b/src/test/rpc_wallet_tests.cpp index e444eb2a5..13ac7de20 100644 --- a/src/test/rpc_wallet_tests.cpp +++ b/src/test/rpc_wallet_tests.cpp @@ -10,10 +10,10 @@ #include "wallet/wallet.h" #include "test/test_bitcoin.h" +#include "test/test_util.h" #include "zcash/Address.hpp" -#include "rpcserver.h" #include "asyncrpcqueue.h" #include "asyncrpcoperation.h" #include "wallet/asyncrpcoperation_sendmany.h" @@ -32,14 +32,12 @@ #include #include #include +#include #include using namespace std; -extern UniValue createArgs(int nRequired, const char* address1 = NULL, const char* address2 = NULL); -extern UniValue CallRPC(string args); - extern CWallet* pwalletMain; bool find_error(const UniValue& objError, const std::string& expected) { diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 0f3ebe053..b27dfc90a 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -14,6 +14,7 @@ #include "script/sign.h" #include "util.h" #include "test/test_bitcoin.h" +#include "test/test_util.h" #if defined(HAVE_CONSENSUS_LIB) #include "script/zcashconsensus.h" @@ -36,22 +37,6 @@ using namespace std; static const unsigned int flags = SCRIPT_VERIFY_P2SH | SCRIPT_VERIFY_STRICTENC; -unsigned int ParseScriptFlags(string strFlags); -string FormatScriptFlags(unsigned int flags); - -UniValue -read_json(const std::string& jsondata) -{ - UniValue v; - - if (!v.read(jsondata) || !v.isArray()) - { - BOOST_ERROR("Parse error."); - return UniValue(UniValue::VARR); - } - return v.get_array(); -} - BOOST_FIXTURE_TEST_SUITE(script_tests, BasicTestingSetup) CMutableTransaction BuildCreditingTransaction(const CScript& scriptPubKey) diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 989ed7428..a453bc5c5 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -10,6 +10,7 @@ #include "script/script.h" #include "serialize.h" #include "test/test_bitcoin.h" +#include "test/test_util.h" #include "util.h" #include "version.h" #include "sodium.h" @@ -20,8 +21,6 @@ #include -extern UniValue read_json(const std::string& jsondata); - // Old script.cpp SignatureHash function uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, unsigned int nIn, int nHashType) { diff --git a/src/test/test_util.cpp b/src/test/test_util.cpp new file mode 100644 index 000000000..7cc538352 --- /dev/null +++ b/src/test/test_util.cpp @@ -0,0 +1,109 @@ +#include "test_util.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "script/interpreter.h" +#include "rpcclient.h" +#include "rpcserver.h" + +static std::map mapFlagNames = boost::assign::map_list_of + (std::string("NONE"), (unsigned int)SCRIPT_VERIFY_NONE) + (std::string("P2SH"), (unsigned int)SCRIPT_VERIFY_P2SH) + (std::string("STRICTENC"), (unsigned int)SCRIPT_VERIFY_STRICTENC) + (std::string("LOW_S"), (unsigned int)SCRIPT_VERIFY_LOW_S) + (std::string("SIGPUSHONLY"), (unsigned int)SCRIPT_VERIFY_SIGPUSHONLY) + (std::string("MINIMALDATA"), (unsigned int)SCRIPT_VERIFY_MINIMALDATA) + (std::string("NULLDUMMY"), (unsigned int)SCRIPT_VERIFY_NULLDUMMY) + (std::string("DISCOURAGE_UPGRADABLE_NOPS"), (unsigned int)SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS) + (std::string("CLEANSTACK"), (unsigned int)SCRIPT_VERIFY_CLEANSTACK) + (std::string("CHECKLOCKTIMEVERIFY"), (unsigned int)SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY); + +UniValue +read_json(const std::string& jsondata) +{ + UniValue v; + + if (!v.read(jsondata) || !v.isArray()) + { + BOOST_ERROR("Parse error."); + return UniValue(UniValue::VARR); + } + return v.get_array(); +} + +unsigned int ParseScriptFlags(std::string strFlags) +{ + if (strFlags.empty()) { + return 0; + } + unsigned int flags = 0; + std::vector words; + boost::algorithm::split(words, strFlags, boost::algorithm::is_any_of(",")); + + BOOST_FOREACH(std::string word, words) + { + if (!mapFlagNames.count(word)) + BOOST_ERROR("Bad test: unknown verification flag '" << word << "'"); + flags |= mapFlagNames[word]; + } + + return flags; +} + +std::string FormatScriptFlags(unsigned int flags) +{ + if (flags == 0) { + return ""; + } + std::string ret; + std::map::const_iterator it = mapFlagNames.begin(); + while (it != mapFlagNames.end()) { + if (flags & it->second) { + ret += it->first + ","; + } + it++; + } + return ret.substr(0, ret.size() - 1); +} + +UniValue +createArgs(int nRequired, const char* address1, const char* address2) +{ + UniValue result(UniValue::VARR); + result.push_back(nRequired); + UniValue addresses(UniValue::VARR); + if (address1) addresses.push_back(address1); + if (address2) addresses.push_back(address2); + result.push_back(addresses); + return result; +} + +UniValue CallRPC(std::string args) +{ + std::vector vArgs; + boost::split(vArgs, args, boost::is_any_of(" \t")); + std::string strMethod = vArgs[0]; + vArgs.erase(vArgs.begin()); + // Handle empty strings the same way as CLI + for (auto i = 0; i < vArgs.size(); i++) { + if (vArgs[i] == "\"\"") { + vArgs[i] = ""; + } + } + UniValue params = RPCConvertValues(strMethod, vArgs); + + rpcfn_type method = tableRPC[strMethod]->actor; + try { + UniValue result = (*method)(params, false); + return result; + } + catch (const UniValue& objError) { + throw std::runtime_error(find_value(objError, "message").get_str()); + } +} diff --git a/src/test/test_util.h b/src/test/test_util.h new file mode 100644 index 000000000..acad077bf --- /dev/null +++ b/src/test/test_util.h @@ -0,0 +1,15 @@ +#ifndef BITCOIN_TEST_TEST_UTIL_H +#define BITCOIN_TEST_TEST_UTIL_H + +#include + +#include + +UniValue read_json(const std::string& jsondata); +unsigned int ParseScriptFlags(std::string strFlags); +std::string FormatScriptFlags(unsigned int flags); +UniValue createArgs(int nRequired, const char* address1 = NULL, const char* address2 = NULL); +UniValue CallRPC(std::string args); + + +#endif diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 6b207f643..4768e7604 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -15,6 +15,7 @@ #include "main.h" #include "script/script.h" #include "script/script_error.h" +#include "test/test_util.h" #include "primitives/transaction.h" #include "sodium.h" @@ -22,11 +23,7 @@ #include #include -#include -#include -#include #include -#include #include @@ -36,56 +33,6 @@ using namespace std; -// In script_tests.cpp -extern UniValue read_json(const std::string& jsondata); - -static std::map mapFlagNames = boost::assign::map_list_of - (string("NONE"), (unsigned int)SCRIPT_VERIFY_NONE) - (string("P2SH"), (unsigned int)SCRIPT_VERIFY_P2SH) - (string("STRICTENC"), (unsigned int)SCRIPT_VERIFY_STRICTENC) - (string("LOW_S"), (unsigned int)SCRIPT_VERIFY_LOW_S) - (string("SIGPUSHONLY"), (unsigned int)SCRIPT_VERIFY_SIGPUSHONLY) - (string("MINIMALDATA"), (unsigned int)SCRIPT_VERIFY_MINIMALDATA) - (string("NULLDUMMY"), (unsigned int)SCRIPT_VERIFY_NULLDUMMY) - (string("DISCOURAGE_UPGRADABLE_NOPS"), (unsigned int)SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS) - (string("CLEANSTACK"), (unsigned int)SCRIPT_VERIFY_CLEANSTACK) - (string("CHECKLOCKTIMEVERIFY"), (unsigned int)SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY); - -unsigned int ParseScriptFlags(string strFlags) -{ - if (strFlags.empty()) { - return 0; - } - unsigned int flags = 0; - vector words; - boost::algorithm::split(words, strFlags, boost::algorithm::is_any_of(",")); - - BOOST_FOREACH(string word, words) - { - if (!mapFlagNames.count(word)) - BOOST_ERROR("Bad test: unknown verification flag '" << word << "'"); - flags |= mapFlagNames[word]; - } - - return flags; -} - -string FormatScriptFlags(unsigned int flags) -{ - if (flags == 0) { - return ""; - } - string ret; - std::map::const_iterator it = mapFlagNames.begin(); - while (it != mapFlagNames.end()) { - if (flags & it->second) { - ret += it->first + ","; - } - it++; - } - return ret.substr(0, ret.size() - 1); -} - BOOST_FIXTURE_TEST_SUITE(transaction_tests, JoinSplitTestingSetup) BOOST_AUTO_TEST_CASE(tx_valid) From 4109ee36b57225d6a2bd6802785d33ad20dd2a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Sat, 6 Jan 2018 09:51:11 +0100 Subject: [PATCH 003/725] Add missing #includes to test_block.cpp --- src/gtest/test_block.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gtest/test_block.cpp b/src/gtest/test_block.cpp index a0cdc1162..46639c558 100644 --- a/src/gtest/test_block.cpp +++ b/src/gtest/test_block.cpp @@ -1,6 +1,8 @@ #include #include "primitives/block.h" +#include "streams.h" +#include "version.h" TEST(block_tests, header_size_is_expected) { From 5c4a47cbf84c9c5ea805990801d09800c863bbc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Sun, 7 Jan 2018 16:11:33 +0100 Subject: [PATCH 004/725] Add actual header file for utilities in gtest/utils.cpp It's not best practice to not use headers. --- src/Makefile.gtest.include | 1 + src/gtest/test_random.cpp | 4 +--- src/gtest/test_transaction.cpp | 3 +-- src/gtest/utils.cpp | 2 ++ src/gtest/utils.h | 7 +++++++ 5 files changed, 12 insertions(+), 5 deletions(-) create mode 100644 src/gtest/utils.h diff --git a/src/Makefile.gtest.include b/src/Makefile.gtest.include index 522bfa84a..0002981da 100644 --- a/src/Makefile.gtest.include +++ b/src/Makefile.gtest.include @@ -7,6 +7,7 @@ noinst_PROGRAMS += zcash-gtest zcash_gtest_SOURCES = \ gtest/main.cpp \ gtest/utils.cpp \ + gtest/utils.h \ gtest/test_checktransaction.cpp \ gtest/json_test_vectors.cpp \ gtest/json_test_vectors.h \ diff --git a/src/gtest/test_random.cpp b/src/gtest/test_random.cpp index d89702bcd..9ae2a6d82 100644 --- a/src/gtest/test_random.cpp +++ b/src/gtest/test_random.cpp @@ -1,9 +1,7 @@ #include #include "random.h" - -extern int GenZero(int n); -extern int GenMax(int n); +#include "gtest/utils.h" TEST(Random, MappedShuffle) { std::vector a {8, 4, 6, 3, 5}; diff --git a/src/gtest/test_transaction.cpp b/src/gtest/test_transaction.cpp index fb68fd35c..c80ed2b14 100644 --- a/src/gtest/test_transaction.cpp +++ b/src/gtest/test_transaction.cpp @@ -1,12 +1,11 @@ #include +#include "gtest/utils.h" #include "primitives/transaction.h" #include "zcash/Note.hpp" #include "zcash/Address.hpp" extern ZCJoinSplit* params; -extern int GenZero(int n); -extern int GenMax(int n); TEST(Transaction, JSDescriptionRandomized) { // construct a merkle tree diff --git a/src/gtest/utils.cpp b/src/gtest/utils.cpp index 527e613e2..30d452c82 100644 --- a/src/gtest/utils.cpp +++ b/src/gtest/utils.cpp @@ -1,3 +1,5 @@ +#include "gtest/utils.h" + int GenZero(int n) { return 0; diff --git a/src/gtest/utils.h b/src/gtest/utils.h new file mode 100644 index 000000000..d3e433ff8 --- /dev/null +++ b/src/gtest/utils.h @@ -0,0 +1,7 @@ +#ifndef BITCOIN_GTEST_UTILS_H +#define BITCOIN_GTEST_UTILS_H + +int GenZero(int n); +int GenMax(int n); + +#endif From f9200ca9c21404373f1e370b0a04afdd66c7b931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Sun, 7 Jan 2018 16:25:51 +0100 Subject: [PATCH 005/725] Fix linkage issue with consts in primitives/block.h static does not mean the same thing when it's used within a class. --- src/primitives/block.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/primitives/block.h b/src/primitives/block.h index 6b3f13a86..21a8d0085 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -21,8 +21,8 @@ class CBlockHeader { public: // header - static const size_t HEADER_SIZE=4+32+32+32+4+4+32; // excluding Equihash solution - static const int32_t CURRENT_VERSION=4; + enum : size_t { HEADER_SIZE=4+32+32+32+4+4+32 }; // excluding Equihash solution + enum : int32_t { CURRENT_VERSION=4 }; int32_t nVersion; uint256 hashPrevBlock; uint256 hashMerkleRoot; From 2d8ee6a68aeff597a4d234d6152bf9b4c4523356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Wed, 10 Jan 2018 22:27:18 +0100 Subject: [PATCH 006/725] Remove Checkpoints_tests.cpp These have been removed "for now" for two years now. I care about this because the way Bazel runs the tests (one executable per test .cpp file) makes empty test suites like this fail. --- src/Makefile.test.include | 1 - src/test/Checkpoints_tests.cpp | 29 ----------------------------- 2 files changed, 30 deletions(-) delete mode 100644 src/test/Checkpoints_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 7c90e8754..45536da53 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -51,7 +51,6 @@ BITCOIN_TESTS =\ test/bip32_tests.cpp \ test/bloom_tests.cpp \ test/checkblock_tests.cpp \ - test/Checkpoints_tests.cpp \ test/coins_tests.cpp \ test/compress_tests.cpp \ test/crypto_tests.cpp \ diff --git a/src/test/Checkpoints_tests.cpp b/src/test/Checkpoints_tests.cpp deleted file mode 100644 index 32b14cb93..000000000 --- a/src/test/Checkpoints_tests.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2011-2013 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -// -// Unit tests for block-chain checkpoints -// - -#include "checkpoints.h" - -#include "uint256.h" -#include "test/test_bitcoin.h" -#include "chainparams.h" - -#include - -using namespace std; - -BOOST_FIXTURE_TEST_SUITE(Checkpoints_tests, BasicTestingSetup) - -// TODO: checkpoints have been removed for now. -/* -BOOST_AUTO_TEST_CASE(sanity) -{ - const CCheckpointData& checkpoints = Params(CBaseChainParams::MAIN).Checkpoints(); - BOOST_CHECK(Checkpoints::GetTotalBlocksEstimate(checkpoints) >= 134444); -} -*/ -BOOST_AUTO_TEST_SUITE_END() From 9b006ce9e875805376657366ff71d88a5e0a222d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Sat, 13 Jan 2018 12:57:06 +0100 Subject: [PATCH 007/725] libsnark: Don't (implicitly) rely on other tests initializing the public params If the other tests are not run, these tests crash with a segfault. --- .../relations/arithmetic_programs/qap/tests/test_qap.cpp | 2 ++ .../ppzksnark/r1cs_ppzksnark/tests/test_r1cs_ppzksnark.cpp | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/snark/libsnark/relations/arithmetic_programs/qap/tests/test_qap.cpp b/src/snark/libsnark/relations/arithmetic_programs/qap/tests/test_qap.cpp index e20f589c9..9f43bb87f 100644 --- a/src/snark/libsnark/relations/arithmetic_programs/qap/tests/test_qap.cpp +++ b/src/snark/libsnark/relations/arithmetic_programs/qap/tests/test_qap.cpp @@ -86,6 +86,8 @@ void test_qap(const size_t qap_degree, const size_t num_inputs, const bool binar TEST(relations, qap) { + alt_bn128_pp::init_public_params(); + start_profiling(); const size_t num_inputs = 10; diff --git a/src/snark/libsnark/zk_proof_systems/ppzksnark/r1cs_ppzksnark/tests/test_r1cs_ppzksnark.cpp b/src/snark/libsnark/zk_proof_systems/ppzksnark/r1cs_ppzksnark/tests/test_r1cs_ppzksnark.cpp index 6c6e51857..9db598045 100644 --- a/src/snark/libsnark/zk_proof_systems/ppzksnark/r1cs_ppzksnark/tests/test_r1cs_ppzksnark.cpp +++ b/src/snark/libsnark/zk_proof_systems/ppzksnark/r1cs_ppzksnark/tests/test_r1cs_ppzksnark.cpp @@ -38,6 +38,8 @@ void test_r1cs_ppzksnark(size_t num_constraints, TEST(zk_proof_systems, r1cs_ppzksnark) { + alt_bn128_pp::init_public_params(); + start_profiling(); test_r1cs_ppzksnark(1000, 20); From 86627fed8e618a4e15989a3e151d7199dccb8eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Sun, 14 Jan 2018 22:22:57 +0100 Subject: [PATCH 008/725] Add missing libsnark initialization call Without it, the sighash_tests test fails when run on its own. --- src/zcash/Proof.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/zcash/Proof.cpp b/src/zcash/Proof.cpp index e7264e684..853b93628 100644 --- a/src/zcash/Proof.cpp +++ b/src/zcash/Proof.cpp @@ -202,6 +202,7 @@ r1cs_ppzksnark_proof ZCProof::to_libsnark_proof() const ZCProof ZCProof::random_invalid() { + initialize_curve_params(); ZCProof p; p.g_A = curve_G1::random_element(); p.g_A_prime = curve_G1::random_element(); From f56fc01fc996d4bac48b5f9e10a17dc58661206b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Sun, 14 Jan 2018 23:56:16 +0100 Subject: [PATCH 009/725] Don't clobber cwd in rpc_wallet_tests.cpp When the tests are run with Bazel, they open the verifier key using a relative path for each test, so changing the cwd will break subsequent tests. --- src/test/rpc_wallet_tests.cpp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/test/rpc_wallet_tests.cpp b/src/test/rpc_wallet_tests.cpp index 13ac7de20..09b5a15a9 100644 --- a/src/test/rpc_wallet_tests.cpp +++ b/src/test/rpc_wallet_tests.cpp @@ -40,10 +40,29 @@ using namespace std; extern CWallet* pwalletMain; +namespace { + bool find_error(const UniValue& objError, const std::string& expected) { return find_value(objError, "message").get_str().find(expected) != string::npos; } +/** Set the working directory for the duration of the scope. */ +class PushCurrentDirectory { +public: + PushCurrentDirectory(const std::string &new_cwd) + : old_cwd(boost::filesystem::current_path()) { + boost::filesystem::current_path(new_cwd); + } + + ~PushCurrentDirectory() { + boost::filesystem::current_path(old_cwd); + } +private: + boost::filesystem::path old_cwd; +}; + +} + BOOST_FIXTURE_TEST_SUITE(rpc_wallet_tests, TestingSetup) BOOST_AUTO_TEST_CASE(rpc_addmultisig) @@ -1212,7 +1231,7 @@ BOOST_AUTO_TEST_CASE(rpc_wallet_encrypted_wallet_zkeys) strWalletPass.reserve(100); strWalletPass = "hello"; - boost::filesystem::current_path(GetArg("-datadir","/tmp/thisshouldnothappen")); + PushCurrentDirectory push_dir(GetArg("-datadir","/tmp/thisshouldnothappen")); BOOST_CHECK(pwalletMain->EncryptWallet(strWalletPass)); // Verify we can still list the keys imported From 39d87d8f16274cb207124bd911ef0718bdb2ade5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Mon, 22 Jan 2018 08:13:11 +0100 Subject: [PATCH 010/725] Include header files within the source tree using "" instead of <> The <> syntax is intended for system header files, see https://gcc.gnu.org/onlinedocs/cpp/Include-Syntax.html I think <> might make sense to use when including external libraries but files that are under src/ of this repo really don't fall under that category IMO. The reason I care is that although Bazel can be made to work with this with some hacking, the normal way of specifying include paths does not allow the <> syntax if the file is in the same WORKSPACE. --- src/init.cpp | 2 +- src/leveldbwrapper.cpp | 8 ++++---- src/leveldbwrapper.h | 4 ++-- src/pubkey.cpp | 4 ++-- src/zcash/JoinSplit.cpp | 8 ++++---- src/zcash/Proof.cpp | 4 ++-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 7df31ab1b..fe7f17009 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -53,7 +53,7 @@ #include #include -#include +#include "libsnark/common/profiling.hpp" #if ENABLE_ZMQ #include "zmq/zmqnotificationinterface.h" diff --git a/src/leveldbwrapper.cpp b/src/leveldbwrapper.cpp index b5d024abb..9185e149e 100644 --- a/src/leveldbwrapper.cpp +++ b/src/leveldbwrapper.cpp @@ -8,10 +8,10 @@ #include -#include -#include -#include -#include +#include "leveldb/cache.h" +#include "leveldb/env.h" +#include "leveldb/filter_policy.h" +#include "memenv.h" void HandleError(const leveldb::Status& status) { diff --git a/src/leveldbwrapper.h b/src/leveldbwrapper.h index 639f736a5..3c1a39c38 100644 --- a/src/leveldbwrapper.h +++ b/src/leveldbwrapper.h @@ -13,8 +13,8 @@ #include -#include -#include +#include "leveldb/db.h" +#include "leveldb/write_batch.h" class leveldb_error : public std::runtime_error { diff --git a/src/pubkey.cpp b/src/pubkey.cpp index 0b87bb526..f404fed5a 100644 --- a/src/pubkey.cpp +++ b/src/pubkey.cpp @@ -5,8 +5,8 @@ #include "pubkey.h" -#include -#include +#include "secp256k1.h" +#include "secp256k1_recovery.h" namespace { diff --git a/src/zcash/JoinSplit.cpp b/src/zcash/JoinSplit.cpp index 2685569d3..ab2059238 100644 --- a/src/zcash/JoinSplit.cpp +++ b/src/zcash/JoinSplit.cpp @@ -10,10 +10,10 @@ #include #include #include -#include -#include -#include -#include +#include "libsnark/common/default_types/r1cs_ppzksnark_pp.hpp" +#include "libsnark/zk_proof_systems/ppzksnark/r1cs_ppzksnark/r1cs_ppzksnark.hpp" +#include "libsnark/gadgetlib1/gadgets/hashes/sha256/sha256_gadget.hpp" +#include "libsnark/gadgetlib1/gadgets/merkle_tree/merkle_tree_check_read_gadget.hpp" #include "tinyformat.h" #include "sync.h" #include "amount.h" diff --git a/src/zcash/Proof.cpp b/src/zcash/Proof.cpp index 853b93628..c41d12628 100644 --- a/src/zcash/Proof.cpp +++ b/src/zcash/Proof.cpp @@ -3,8 +3,8 @@ #include "crypto/common.h" #include -#include -#include +#include "libsnark/common/default_types/r1cs_ppzksnark_pp.hpp" +#include "libsnark/zk_proof_systems/ppzksnark/r1cs_ppzksnark/r1cs_ppzksnark.hpp" #include using namespace libsnark; From 2ee9ce678640ed4bd72d011b43c3397de2ea1a08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Mon, 22 Jan 2018 07:41:21 +0100 Subject: [PATCH 011/725] Be consistent about what path to include bitcoin-config.h with Adding unnecessary include search paths is a sure way to run into trouble. This matters for Bazel because there, bitcoin-config.h is not actually generated within the source tree. --- src/config/.empty | 0 src/crypto/common.h | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 src/config/.empty diff --git a/src/config/.empty b/src/config/.empty deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/crypto/common.h b/src/crypto/common.h index ad4c6dd5e..faaafd760 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -6,7 +6,7 @@ #define BITCOIN_CRYPTO_COMMON_H #if defined(HAVE_CONFIG_H) -#include "bitcoin-config.h" +#include "config/bitcoin-config.h" #endif #include From 4bffaf040c29cec08235b07c6d2e6f9b8024580b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Per=20Gr=C3=B6n?= Date: Mon, 22 Jan 2018 08:21:04 +0100 Subject: [PATCH 012/725] Be consistent with how to #include test data headers It helps the Bazel scripts to not have multiple search paths to a single header, and it seems to be the right thing to do anyway. --- src/test/alert_tests.cpp | 2 +- src/test/base58_tests.cpp | 6 +++--- src/test/script_tests.cpp | 4 ++-- src/test/sighash_tests.cpp | 2 +- src/test/transaction_tests.cpp | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/alert_tests.cpp b/src/test/alert_tests.cpp index ab3841c48..0bd8a2f39 100644 --- a/src/test/alert_tests.cpp +++ b/src/test/alert_tests.cpp @@ -10,7 +10,7 @@ #include "chain.h" #include "chainparams.h" #include "clientversion.h" -#include "data/alertTests.raw.h" +#include "test/data/alertTests.raw.h" #include "main.h" #include "rpcprotocol.h" diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index ec159ed28..b6530ec36 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -4,9 +4,9 @@ #include "base58.h" -#include "data/base58_encode_decode.json.h" -#include "data/base58_keys_invalid.json.h" -#include "data/base58_keys_valid.json.h" +#include "test/data/base58_encode_decode.json.h" +#include "test/data/base58_keys_invalid.json.h" +#include "test/data/base58_keys_valid.json.h" #include "key.h" #include "script/script.h" diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index b27dfc90a..24acb714e 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "data/script_invalid.json.h" -#include "data/script_valid.json.h" +#include "test/data/script_invalid.json.h" +#include "test/data/script_valid.json.h" #include "core_io.h" #include "key.h" diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index a453bc5c5..130e4c6a3 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "consensus/validation.h" -#include "data/sighash.json.h" +#include "test/data/sighash.json.h" #include "main.h" #include "random.h" #include "script/interpreter.h" diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 4768e7604..771fc41bd 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include "data/tx_invalid.json.h" -#include "data/tx_valid.json.h" +#include "test/data/tx_invalid.json.h" +#include "test/data/tx_valid.json.h" #include "test/test_bitcoin.h" #include "init.h" From 9c96affb0d9874ef1ecb283b3b84baca21e7a393 Mon Sep 17 00:00:00 2001 From: LongShao007 <007longshao@gmail.com> Date: Sun, 14 Apr 2019 18:16:55 +0800 Subject: [PATCH 013/725] fix bug of bdb.mk --- depends/packages/bdb.mk | 1 - 1 file changed, 1 deletion(-) diff --git a/depends/packages/bdb.mk b/depends/packages/bdb.mk index 6b385f2ab..b8ec71112 100644 --- a/depends/packages/bdb.mk +++ b/depends/packages/bdb.mk @@ -14,7 +14,6 @@ endef define $(package)_preprocess_cmds sed -i.old 's/WinIoCtl.h/winioctl.h/g' src/dbinc/win_db.h && \ - sed -i.old 's/__atomic_compare_exchange\\(/__atomic_compare_exchange_db(/' src/dbinc/atomic.h && \ sed -i.old 's/atomic_init/atomic_init_db/' src/dbinc/atomic.h src/mp/mp_region.c src/mp/mp_mvcc.c src/mp/mp_fget.c src/mutex/mut_method.c src/mutex/mut_tas.c endef From bb0d3acb87ca5e4b08ce0c7897dad2268a390c82 Mon Sep 17 00:00:00 2001 From: arielgabizon Date: Fri, 12 Oct 2018 22:44:04 +0200 Subject: [PATCH 014/725] explain expiry error --- src/main.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index 5ac46cb7c..79e6b7e24 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -871,7 +871,10 @@ bool ContextualCheckTransaction( if (IsExpiredTx(tx, nHeight)) { // Don't increase banscore if the transaction only just expired int expiredDosLevel = IsExpiredTx(tx, nHeight - 1) ? dosLevelConstricting : 0; - return state.DoS(expiredDosLevel, error("ContextualCheckTransaction(): transaction is expired"), REJECT_INVALID, "tx-overwinter-expired"); + return state.DoS( + expiredDosLevel, + error("ContextualCheckTransaction(): transaction is expired. Resending when caught up with the blockchain, or manually setting the zcashd txexpirydelta parameter may help."), + REJECT_INVALID, "tx-overwinter-expired"); } } From 4b61aede289cd27b7aed34a544e1963aebc3e75d Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 2 Jan 2020 11:04:47 -0300 Subject: [PATCH 015/725] add bool argument to get balance in satoshis to rpc getreceivedbyaddress --- qa/rpc-tests/receivedby.py | 5 +++++ src/wallet/rpcwallet.cpp | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/receivedby.py b/qa/rpc-tests/receivedby.py index b19c55e68..dc17bf088 100755 --- a/qa/rpc-tests/receivedby.py +++ b/qa/rpc-tests/receivedby.py @@ -110,6 +110,11 @@ class ReceivedByTest(BitcoinTestFramework): if balance != Decimal("0.1"): raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + # Get balance as integer + balance = self.nodes[1].getreceivedbyaddress(addr, 0, True) + if balance != int("10000000"): + raise AssertionError("Wrong balance returned by getreceivedbyaddress, %i"%(balance)) + ''' listreceivedbyaccount + getreceivedbyaccount Test ''' diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index c08da799c..6dae48435 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -597,7 +597,7 @@ UniValue getreceivedbyaddress(const UniValue& params, bool fHelp) if (!EnsureWalletIsAvailable(fHelp)) return NullUniValue; - if (fHelp || params.size() < 1 || params.size() > 2) + if (fHelp || params.size() < 1 || params.size() > 3) throw runtime_error( "getreceivedbyaddress \"zcashaddress\" ( minconf )\n" "\nReturns the total amount received by the given Zcash address in transactions with at least minconf confirmations.\n" @@ -647,8 +647,11 @@ UniValue getreceivedbyaddress(const UniValue& params, bool fHelp) if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; } + if(params.size() > 2) + if(params[2].get_bool()) + return nAmount; - return ValueFromAmount(nAmount); + return ValueFromAmount(nAmount); } From aba1d946733749c232750349d5ed96a5f0b02b5c Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Thu, 2 Jan 2020 11:38:38 -0300 Subject: [PATCH 016/725] add documentation to flag --- src/wallet/rpcwallet.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 6dae48435..3cc6709b5 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -603,7 +603,8 @@ UniValue getreceivedbyaddress(const UniValue& params, bool fHelp) "\nReturns the total amount received by the given Zcash address in transactions with at least minconf confirmations.\n" "\nArguments:\n" "1. \"zcashaddress\" (string, required) The Zcash address for transactions.\n" - "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" + "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" + "3. inzatoshi (bool, optional, default=false) Get the result amount as an integer.\n" "\nResult:\n" "amount (numeric) The total amount in " + CURRENCY_UNIT + " received at this address.\n" "\nExamples:\n" From 4155db0325525dc5f376a1be13ac6ec66561f2b5 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 28 Jan 2020 11:11:43 -0300 Subject: [PATCH 017/725] change argument name --- src/wallet/rpcwallet.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 3cc6709b5..dd57a8e22 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -604,7 +604,7 @@ UniValue getreceivedbyaddress(const UniValue& params, bool fHelp) "\nArguments:\n" "1. \"zcashaddress\" (string, required) The Zcash address for transactions.\n" "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" - "3. inzatoshi (bool, optional, default=false) Get the result amount as an integer.\n" + "3. inZat (bool, optional, default=false) Get the result amount as an integer.\n" "\nResult:\n" "amount (numeric) The total amount in " + CURRENCY_UNIT + " received at this address.\n" "\nExamples:\n" @@ -648,8 +648,10 @@ UniValue getreceivedbyaddress(const UniValue& params, bool fHelp) if (wtx.GetDepthInMainChain() >= nMinDepth) nAmount += txout.nValue; } - if(params.size() > 2) - if(params[2].get_bool()) + + // inZat + if (params.size() > 2) + if (params[2].get_bool()) return nAmount; return ValueFromAmount(nAmount); From d2c26b33e01ab7c7fd1b8c830244031e274c6cd0 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 28 Jan 2020 12:48:40 -0300 Subject: [PATCH 018/725] add boolean inZat to getreceivedbyaccount --- qa/rpc-tests/receivedby.py | 11 ++++++++--- src/wallet/rpcwallet.cpp | 14 ++++++++++---- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/qa/rpc-tests/receivedby.py b/qa/rpc-tests/receivedby.py index dc17bf088..3184adf68 100755 --- a/qa/rpc-tests/receivedby.py +++ b/qa/rpc-tests/receivedby.py @@ -111,7 +111,7 @@ class ReceivedByTest(BitcoinTestFramework): raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) # Get balance as integer - balance = self.nodes[1].getreceivedbyaddress(addr, 0, True) + balance = self.nodes[1].getreceivedbyaddress(addr, 1, True) if balance != int("10000000"): raise AssertionError("Wrong balance returned by getreceivedbyaddress, %i"%(balance)) @@ -134,7 +134,7 @@ class ReceivedByTest(BitcoinTestFramework): {"account":account}, received_by_account_json) - # getreceivedbyaddress should return same balance because of 0 confirmations + # getreceivedbyaccount should return same balance because of 0 confirmations balance = self.nodes[1].getreceivedbyaccount(account) if balance != balance_by_account: raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) @@ -146,11 +146,16 @@ class ReceivedByTest(BitcoinTestFramework): {"account":account}, {"account":received_by_account_json["account"], "amount":(received_by_account_json["amount"] + Decimal("0.1"))}) - # getreceivedbyaddress should return updates balance + # getreceivedbyaccount should return updates balance balance = self.nodes[1].getreceivedbyaccount(account) if balance != balance_by_account + Decimal("0.1"): raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + # Get balance as integer + balance = self.nodes[1].getreceivedbyaccount(account, 1, True) + if balance != int("30000000"): + raise AssertionError("Wrong balance returned by getreceivedbyaccount, %i"%(balance)) + # Create a new account named "mynewaccount" that has a 0 balance self.nodes[1].getaccountaddress("mynewaccount") received_by_account_json = get_sub_array_from_array(self.nodes[1].listreceivedbyaccount(0,True),{"account":"mynewaccount"}) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index dd57a8e22..c822432f8 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -599,7 +599,7 @@ UniValue getreceivedbyaddress(const UniValue& params, bool fHelp) if (fHelp || params.size() < 1 || params.size() > 3) throw runtime_error( - "getreceivedbyaddress \"zcashaddress\" ( minconf )\n" + "getreceivedbyaddress \"zcashaddress\" ( minconf ) ( inZat )\n" "\nReturns the total amount received by the given Zcash address in transactions with at least minconf confirmations.\n" "\nArguments:\n" "1. \"zcashaddress\" (string, required) The Zcash address for transactions.\n" @@ -663,13 +663,14 @@ UniValue getreceivedbyaccount(const UniValue& params, bool fHelp) if (!EnsureWalletIsAvailable(fHelp)) return NullUniValue; - if (fHelp || params.size() < 1 || params.size() > 2) + if (fHelp || params.size() < 1 || params.size() > 3) throw runtime_error( - "getreceivedbyaccount \"account\" ( minconf )\n" + "getreceivedbyaccount \"account\" ( minconf ) ( inZat )\n" "\nDEPRECATED. Returns the total amount received by addresses with in transactions with at least [minconf] confirmations.\n" "\nArguments:\n" "1. \"account\" (string, required) MUST be set to the empty string \"\" to represent the default account. Passing any other string will result in an error.\n" - "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" + "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" + "3. inZat (bool, optional, default=false) Get the result amount as an integer.\n" "\nResult:\n" "amount (numeric) The total amount in " + CURRENCY_UNIT + " received for this account.\n" "\nExamples:\n" @@ -711,6 +712,11 @@ UniValue getreceivedbyaccount(const UniValue& params, bool fHelp) } } + // inZat + if (params.size() > 2) + if (params[2].get_bool()) + return nAmount; + return ValueFromAmount(nAmount); } From 50372cab6250757dfc70f8e3a35ca2a5f3fbc260 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 28 Jan 2020 17:36:26 -0300 Subject: [PATCH 019/725] add boolean inZat to getbalance --- qa/rpc-tests/wallet.py | 5 ++++- src/wallet/rpcwallet.cpp | 11 +++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index ba4c7afdf..57fdf1e51 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -263,10 +263,13 @@ class WalletTest (BitcoinTestFramework): self.nodes[0].generate(1) sync_blocks(self.nodes) - #tx should be added to balance because after restarting the nodes tx should be broadcastet + # tx should be added to balance because after restarting the nodes tx should be broadcastet assert_equal(self.nodes[2].getbalance(), Decimal('13.99800000')) #should not be assert_equal(self.nodes[2].getbalance("*"), Decimal('13.99800000')) #should not be + # check integer balances from getbalance + assert_equal(self.nodes[2].getbalance("*", 1, False, True), int('1399800000')) #should not be + # send from node 0 to node 2 taddr mytaddr = self.nodes[2].getnewaddress() mytxid = self.nodes[0].sendtoaddress(mytaddr, 10.0) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index c822432f8..b55e8ae83 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -758,14 +758,15 @@ UniValue getbalance(const UniValue& params, bool fHelp) if (!EnsureWalletIsAvailable(fHelp)) return NullUniValue; - if (fHelp || params.size() > 3) + if (fHelp || params.size() > 4) throw runtime_error( - "getbalance ( \"account\" minconf includeWatchonly )\n" + "getbalance ( \"account\" minconf includeWatchonly inZat )\n" "\nReturns the server's total available balance.\n" "\nArguments:\n" "1. \"account\" (string, optional) DEPRECATED. If provided, it MUST be set to the empty string \"\" or to the string \"*\", either of which will give the total available balance. Passing any other string will result in an error.\n" "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" "3. includeWatchonly (bool, optional, default=false) Also include balance in watchonly addresses (see 'importaddress')\n" + "4. inZat (bool, optional, default=false) Get the result amount as an integer.\n" "\nResult:\n" "amount (numeric) The total amount in " + CURRENCY_UNIT + " received for this account.\n" "\nExamples:\n" @@ -815,6 +816,12 @@ UniValue getbalance(const UniValue& params, bool fHelp) nBalance -= s.amount; nBalance -= allFee; } + + // inZat + if (params.size() > 3) + if (params[3].get_bool()) + return nBalance; + return ValueFromAmount(nBalance); } From 306270911ecc1ab7e2f88ce1d7632ab0f4698d97 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 28 Jan 2020 17:55:37 -0300 Subject: [PATCH 020/725] add boolean inZat to z_getbalance --- qa/rpc-tests/wallet.py | 3 +++ src/wallet/rpcwallet.cpp | 10 ++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index 57fdf1e51..226ca4994 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -280,6 +280,9 @@ class WalletTest (BitcoinTestFramework): mybalance = self.nodes[2].z_getbalance(mytaddr) assert_equal(mybalance, Decimal('10.0')) + # check integer balances from z_getbalance + assert_equal(self.nodes[2].z_getbalance(mytaddr, 1, True), int('10')) + mytxdetails = self.nodes[2].gettransaction(mytxid) myvjoinsplits = mytxdetails["vjoinsplit"] assert_equal(0, len(myvjoinsplits)) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index b55e8ae83..de05acd4b 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3470,15 +3470,16 @@ UniValue z_getbalance(const UniValue& params, bool fHelp) if (!EnsureWalletIsAvailable(fHelp)) return NullUniValue; - if (fHelp || params.size()==0 || params.size() >2) + if (fHelp || params.size() == 0 || params.size() > 3) throw runtime_error( - "z_getbalance \"address\" ( minconf )\n" + "z_getbalance \"address\" ( minconf inZat )\n" "\nReturns the balance of a taddr or zaddr belonging to the node's wallet.\n" "\nCAUTION: If the wallet has only an incoming viewing key for this address, then spends cannot be" "\ndetected, and so the returned balance may be larger than the actual balance.\n" "\nArguments:\n" "1. \"address\" (string) The selected address. It may be a transparent or private address.\n" "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" + "3. inZat (bool, optional, default=false) Get the result amount as an integer.\n" "\nResult:\n" "amount (numeric) The total amount in " + CURRENCY_UNIT + " received for this address.\n" "\nExamples:\n" @@ -3522,6 +3523,11 @@ UniValue z_getbalance(const UniValue& params, bool fHelp) nBalance = getBalanceZaddr(fromaddress, nMinDepth, false); } + // inZat + if (params.size() > 3) + if (params[3].get_bool()) + return nBalance; + return ValueFromAmount(nBalance); } From 2ec9ac3c81888e57a01c036c1da0b4797a69bdc9 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 28 Jan 2020 18:30:14 -0300 Subject: [PATCH 021/725] add amountZat field to listreceivedbyaddress and listreceivedbyaccount --- qa/rpc-tests/receivedby.py | 8 ++++---- src/wallet/rpcwallet.cpp | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/qa/rpc-tests/receivedby.py b/qa/rpc-tests/receivedby.py index 3184adf68..2c23cb892 100755 --- a/qa/rpc-tests/receivedby.py +++ b/qa/rpc-tests/receivedby.py @@ -71,11 +71,11 @@ class ReceivedByTest(BitcoinTestFramework): self.sync_all() check_array_result(self.nodes[1].listreceivedbyaddress(), {"address":addr}, - {"address":addr, "account":"", "amount":Decimal("0.1"), "confirmations":10, "txids":[txid,]}) + {"address":addr, "account":"", "amount":Decimal("0.1"), "amountZat":int("10000000"), "confirmations":10, "txids":[txid,]}) # With min confidence < 10 check_array_result(self.nodes[1].listreceivedbyaddress(5), {"address":addr}, - {"address":addr, "account":"", "amount":Decimal("0.1"), "confirmations":10, "txids":[txid,]}) + {"address":addr, "account":"", "amount":Decimal("0.1"), "amountZat":int("10000000"), "confirmations":10, "txids":[txid,]}) # With min confidence > 10, should not find Tx check_array_result(self.nodes[1].listreceivedbyaddress(11),{"address":addr},{ },True) @@ -83,7 +83,7 @@ class ReceivedByTest(BitcoinTestFramework): addr = self.nodes[1].getnewaddress() check_array_result(self.nodes[1].listreceivedbyaddress(0,True), {"address":addr}, - {"address":addr, "account":"", "amount":0, "confirmations":0, "txids":[]}) + {"address":addr, "account":"", "amount":0, "confirmations":0, "amountZat":int("0"), "txids":[]}) ''' getreceivedbyaddress Test @@ -144,7 +144,7 @@ class ReceivedByTest(BitcoinTestFramework): # listreceivedbyaccount should return updated account balance check_array_result(self.nodes[1].listreceivedbyaccount(), {"account":account}, - {"account":received_by_account_json["account"], "amount":(received_by_account_json["amount"] + Decimal("0.1"))}) + {"account":received_by_account_json["account"], "amount":(received_by_account_json["amount"] + Decimal("0.1")), "amountZat":int("30000000")}) # getreceivedbyaccount should return updates balance balance = self.nodes[1].getreceivedbyaccount(account) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index de05acd4b..535906362 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1248,6 +1248,7 @@ UniValue ListReceived(const UniValue& params, bool fByAccounts) obj.pushKV("address", EncodeDestination(dest)); obj.pushKV("account", strAccount); obj.pushKV("amount", ValueFromAmount(nAmount)); + obj.pushKV("amountZat", nAmount); obj.pushKV("confirmations", (nConf == std::numeric_limits::max() ? 0 : nConf)); UniValue transactions(UniValue::VARR); if (it != mapTally.end()) @@ -1273,6 +1274,7 @@ UniValue ListReceived(const UniValue& params, bool fByAccounts) obj.pushKV("involvesWatchonly", true); obj.pushKV("account", (*it).first); obj.pushKV("amount", ValueFromAmount(nAmount)); + obj.pushKV("amountZat", nAmount); obj.pushKV("confirmations", (nConf == std::numeric_limits::max() ? 0 : nConf)); ret.push_back(obj); } @@ -1302,6 +1304,7 @@ UniValue listreceivedbyaddress(const UniValue& params, bool fHelp) " \"address\" : \"receivingaddress\", (string) The receiving address\n" " \"account\" : \"accountname\", (string) DEPRECATED. The account of the receiving address. The default account is \"\".\n" " \"amount\" : x.xxx, (numeric) The total amount in " + CURRENCY_UNIT + " received by the address\n" + " \"amountZat\" : xxxx (numeric) The amount in zatoshis\n" " \"confirmations\" : n (numeric) The number of confirmations of the most recent transaction included\n" " }\n" " ,...\n" @@ -1338,6 +1341,7 @@ UniValue listreceivedbyaccount(const UniValue& params, bool fHelp) " \"involvesWatchonly\" : true, (bool) Only returned if imported addresses were involved in transaction\n" " \"account\" : \"accountname\", (string) The account name of the receiving account\n" " \"amount\" : x.xxx, (numeric) The total amount received by addresses with this account\n" + " \"amountZat\" : xxxx (numeric) The amount in zatoshis\n" " \"confirmations\" : n (numeric) The number of confirmations of the most recent transaction included\n" " }\n" " ,...\n" From 9a55c8b23898da7ea32b64d8999bd53ba3e61c82 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 28 Jan 2020 20:34:40 -0300 Subject: [PATCH 022/725] add amountZat field to listtransactions, gettransaction and listsinceblock --- qa/rpc-tests/listtransactions.py | 32 +++++++++++++++++--------------- qa/rpc-tests/wallet.py | 3 +++ src/wallet/rpcwallet.cpp | 8 ++++++++ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/qa/rpc-tests/listtransactions.py b/qa/rpc-tests/listtransactions.py index 00e17a989..5cdc48d04 100755 --- a/qa/rpc-tests/listtransactions.py +++ b/qa/rpc-tests/listtransactions.py @@ -6,6 +6,7 @@ # Exercise the listtransactions API from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal from decimal import Decimal @@ -38,28 +39,29 @@ class ListTransactionsTest(BitcoinTestFramework): self.sync_all() check_array_result(self.nodes[0].listtransactions(), {"txid":txid}, - {"category":"send","account":"","amount":Decimal("-0.1"),"confirmations":0}) + {"category":"send","account":"","amount":Decimal("-0.1"),"amountZat":int("-10000000"),"confirmations":0}) check_array_result(self.nodes[1].listtransactions(), {"txid":txid}, - {"category":"receive","account":"","amount":Decimal("0.1"),"confirmations":0}) + {"category":"receive","account":"","amount":Decimal("0.1"),"amountZat":int("10000000"),"confirmations":0}) + # mine a block, confirmations should change: self.nodes[0].generate(1) self.sync_all() check_array_result(self.nodes[0].listtransactions(), {"txid":txid}, - {"category":"send","account":"","amount":Decimal("-0.1"),"confirmations":1}) + {"category":"send","account":"","amount":Decimal("-0.1"),"amountZat":int("-10000000"),"confirmations":1}) check_array_result(self.nodes[1].listtransactions(), {"txid":txid}, - {"category":"receive","account":"","amount":Decimal("0.1"),"confirmations":1}) + {"category":"receive","account":"","amount":Decimal("0.1"),"amountZat":int("10000000"),"confirmations":1}) # send-to-self: txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) check_array_result(self.nodes[0].listtransactions(), {"txid":txid, "category":"send"}, - {"amount":Decimal("-0.2")}) + {"amount":Decimal("-0.2"),"amountZat":int("-20000000")}) check_array_result(self.nodes[0].listtransactions(), {"txid":txid, "category":"receive"}, - {"amount":Decimal("0.2")}) + {"amount":Decimal("0.2"),"amountZat":int("20000000")}) # sendmany from node1: twice to self, twice to node2: send_to = { self.nodes[0].getnewaddress() : 0.11, @@ -69,28 +71,28 @@ class ListTransactionsTest(BitcoinTestFramework): txid = self.nodes[1].sendmany("", send_to) self.sync_all() check_array_result(self.nodes[1].listtransactions(), - {"category":"send","amount":Decimal("-0.11")}, + {"category":"send","amount":Decimal("-0.11"),"amountZat":int("-11000000")}, {"txid":txid} ) check_array_result(self.nodes[0].listtransactions(), - {"category":"receive","amount":Decimal("0.11")}, + {"category":"receive","amount":Decimal("0.11"),"amountZat":int("11000000")}, {"txid":txid} ) check_array_result(self.nodes[1].listtransactions(), - {"category":"send","amount":Decimal("-0.22")}, + {"category":"send","amount":Decimal("-0.22"),"amountZat":int("-22000000")}, {"txid":txid} ) check_array_result(self.nodes[1].listtransactions(), - {"category":"receive","amount":Decimal("0.22")}, + {"category":"receive","amount":Decimal("0.22"),"amountZat":int("22000000")}, {"txid":txid} ) check_array_result(self.nodes[1].listtransactions(), - {"category":"send","amount":Decimal("-0.33")}, + {"category":"send","amount":Decimal("-0.33"),"amountZat":int("-33000000")}, {"txid":txid} ) check_array_result(self.nodes[0].listtransactions(), - {"category":"receive","amount":Decimal("0.33")}, + {"category":"receive","amount":Decimal("0.33"),"amountZat":int("33000000")}, {"txid":txid, "account" : ""} ) check_array_result(self.nodes[1].listtransactions(), - {"category":"send","amount":Decimal("-0.44")}, + {"category":"send","amount":Decimal("-0.44"),"amountZat":int("-44000000")}, {"txid":txid, "account" : ""} ) check_array_result(self.nodes[1].listtransactions(), - {"category":"receive","amount":Decimal("0.44")}, + {"category":"receive","amount":Decimal("0.44"),"amountZat":int("44000000")}, {"txid":txid, "account" : ""} ) multisig = self.nodes[1].createmultisig(1, [self.nodes[1].getnewaddress()]) @@ -100,7 +102,7 @@ class ListTransactionsTest(BitcoinTestFramework): self.sync_all() assert(len(self.nodes[0].listtransactions("watchonly", 100, 0, False)) == 0) check_array_result(self.nodes[0].listtransactions("watchonly", 100, 0, True), - {"category":"receive","amount":Decimal("0.1")}, + {"category":"receive","amount":Decimal("0.1"),"amountZat":int("10000000")}, {"txid":txid, "account" : "watchonly"} ) if __name__ == '__main__': diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index 226ca4994..91ecc9cf4 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -385,15 +385,18 @@ class WalletTest (BitcoinTestFramework): txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "2") txObj = self.nodes[0].gettransaction(txId) assert_equal(txObj['amount'], Decimal('-2.00000000')) + assert_equal(txObj['amountZat'], int('-200000000')) txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "0.0001") txObj = self.nodes[0].gettransaction(txId) assert_equal(txObj['amount'], Decimal('-0.00010000')) + assert_equal(txObj['amountZat'], int('-10000')) #check if JSON parser can handle scientific notation in strings txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "1e-4") txObj = self.nodes[0].gettransaction(txId) assert_equal(txObj['amount'], Decimal('-0.00010000')) + assert_equal(txObj['amountZat'], int('-10000')) #this should fail errorString = "" diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 535906362..70e423e62 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1389,6 +1389,7 @@ void ListTransactions(const CWalletTx& wtx, const string& strAccount, int nMinDe MaybePushAddress(entry, s.destination); entry.pushKV("category", "send"); entry.pushKV("amount", ValueFromAmount(-s.amount)); + entry.pushKV("amountZat", -s.amount); entry.pushKV("vout", s.vout); entry.pushKV("fee", ValueFromAmount(-nFee)); if (fLong) @@ -1427,6 +1428,7 @@ void ListTransactions(const CWalletTx& wtx, const string& strAccount, int nMinDe entry.pushKV("category", "receive"); } entry.pushKV("amount", ValueFromAmount(r.amount)); + entry.pushKV("amountZat", r.amount); entry.pushKV("vout", r.vout); if (fLong) WalletTxToJSON(wtx, entry); @@ -1448,6 +1450,7 @@ void AcentryToJSON(const CAccountingEntry& acentry, const string& strAccount, Un entry.pushKV("category", "move"); entry.pushKV("time", acentry.nTime); entry.pushKV("amount", ValueFromAmount(acentry.nCreditDebit)); + entry.pushKV("amountZat", acentry.nCreditDebit); entry.pushKV("otheraccount", acentry.strOtherAccount); entry.pushKV("comment", acentry.strComment); ret.push_back(entry); @@ -1484,6 +1487,7 @@ UniValue listtransactions(const UniValue& params, bool fHelp) " \"amount\": x.xxx, (numeric) The amount in " + CURRENCY_UNIT + ". This is negative for the 'send' category, and for the\n" " 'move' category for moves outbound. It is positive for the 'receive' category,\n" " and for the 'move' category for inbound funds.\n" + " \"amountZat\": x.xxx, (numeric) The amount in zatoshis. Negative and positive are the same as 'amount' field.\n" " \"vout\" : n, (numeric) the vout value\n" " \"fee\": x.xxx, (numeric) The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the \n" " 'send' category of transactions.\n" @@ -1681,6 +1685,7 @@ UniValue listsinceblock(const UniValue& params, bool fHelp) " or 'expired'. Available for 'send' and 'receive' category of transactions.\n" " \"amount\": x.xxx, (numeric) The amount in " + CURRENCY_UNIT + ". This is negative for the 'send' category, and for the 'move' category for moves \n" " outbound. It is positive for the 'receive' category, and for the 'move' category for inbound funds.\n" + " \"amountZat\": x.xxx, (numeric) The amount in zatoshis. Negative and positive are the same as 'amount' field.\n" " \"vout\" : n, (numeric) the vout value\n" " \"fee\": x.xxx, (numeric) The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the 'send' category of transactions.\n" " \"confirmations\": n, (numeric) The number of confirmations for the transaction. Available for 'send' and 'receive' category of transactions.\n" @@ -1767,6 +1772,7 @@ UniValue gettransaction(const UniValue& params, bool fHelp) "{\n" " \"status\" : \"mined|waiting|expiringsoon|expired\", (string) The transaction status, can be 'mined', 'waiting', 'expiringsoon' or 'expired'\n" " \"amount\" : x.xxx, (numeric) The transaction amount in " + CURRENCY_UNIT + "\n" + " \"amountZat\" : x (numeric) The amount in zatoshis\n" " \"confirmations\" : n, (numeric) The number of confirmations\n" " \"blockhash\" : \"hash\", (string) The block hash\n" " \"blockindex\" : xx, (numeric) The block index\n" @@ -1780,6 +1786,7 @@ UniValue gettransaction(const UniValue& params, bool fHelp) " \"address\" : \"zcashaddress\", (string) The Zcash address involved in the transaction\n" " \"category\" : \"send|receive\", (string) The category, either 'send' or 'receive'\n" " \"amount\" : x.xxx (numeric) The amount in " + CURRENCY_UNIT + "\n" + " \"amountZat\" : x (numeric) The amount in zatoshis\n" " \"vout\" : n, (numeric) the vout value\n" " }\n" " ,...\n" @@ -1825,6 +1832,7 @@ UniValue gettransaction(const UniValue& params, bool fHelp) CAmount nFee = (wtx.IsFromMe(filter) ? wtx.GetValueOut() - nDebit : 0); entry.pushKV("amount", ValueFromAmount(nNet - nFee)); + entry.pushKV("amountZat", nNet - nFee); if (wtx.IsFromMe(filter)) entry.pushKV("fee", ValueFromAmount(nFee)); From 76e7f21d4a8031100b8b151ddbc6795f7102ba0c Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 28 Jan 2020 20:44:36 -0300 Subject: [PATCH 023/725] add amountZat field to listunspent --- qa/rpc-tests/wallet.py | 1 + src/wallet/rpcwallet.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index 91ecc9cf4..5803c117d 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -220,6 +220,7 @@ class WalletTest (BitcoinTestFramework): if uTx['txid'] == zeroValueTxid: found = True assert_equal(uTx['amount'], Decimal('0.00000000')) + assert_equal(uTx['amountZat'], int('0')) assert(found) #do some -walletbroadcast tests diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 70e423e62..cf43d0e21 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -2411,6 +2411,7 @@ UniValue listunspent(const UniValue& params, bool fHelp) " \"account\" : \"account\", (string) DEPRECATED. The associated account, or \"\" for the default account\n" " \"scriptPubKey\" : \"key\", (string) the script key\n" " \"amount\" : x.xxx, (numeric) the transaction amount in " + CURRENCY_UNIT + "\n" + " \"amountZat\" : xxxx (numeric) the transaction amount in zatoshis\n" " \"confirmations\" : n, (numeric) The number of confirmations\n" " \"redeemScript\" : n (string) The redeemScript if scriptPubKey is P2SH\n" " \"spendable\" : xxx (bool) Whether we have the private keys to spend this output\n" @@ -2486,6 +2487,7 @@ UniValue listunspent(const UniValue& params, bool fHelp) entry.pushKV("scriptPubKey", HexStr(scriptPubKey.begin(), scriptPubKey.end())); entry.pushKV("amount", ValueFromAmount(out.tx->vout[out.i].nValue)); + entry.pushKV("amountZat", out.tx->vout[out.i].nValue); entry.pushKV("confirmations", out.nDepth); entry.pushKV("spendable", out.fSpendable); results.push_back(entry); From 25f48c21ddee7845b0e933c4e7606843f4797d33 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 28 Jan 2020 21:27:31 -0300 Subject: [PATCH 024/725] add amountZat field to z_listreceivedbyaddress --- qa/rpc-tests/wallet_listreceived.py | 5 +++++ src/wallet/rpcwallet.cpp | 3 +++ 2 files changed, 8 insertions(+) diff --git a/qa/rpc-tests/wallet_listreceived.py b/qa/rpc-tests/wallet_listreceived.py index 52dff938a..ad30e9537 100755 --- a/qa/rpc-tests/wallet_listreceived.py +++ b/qa/rpc-tests/wallet_listreceived.py @@ -15,6 +15,7 @@ my_memo = my_memo + '0'*(1024-len(my_memo)) no_memo = 'f6' + ('0'*1022) # see section 5.5 of the protocol spec fee = Decimal('0.0001') +feeZat = Decimal('10000') class ListReceivedTest (BitcoinTestFramework): @@ -106,6 +107,7 @@ class ListReceivedTest (BitcoinTestFramework): assert_equal(1, len(r), "Should have received one (unconfirmed) note") assert_equal(txid, r[0]['txid']) assert_equal(1, r[0]['amount']) + assert_equal(100000000, r[0]['amountZat']) assert_false(r[0]['change'], "Note should not be change") assert_equal(my_memo, r[0]['memo']) assert_equal(0, r[0]['confirmations']) @@ -203,11 +205,13 @@ class ListReceivedTest (BitcoinTestFramework): assert_equal(txid, r[0]['txid']) assert_equal(Decimal('0.4')-fee, r[0]['amount']) + assert_equal(Decimal('40000000')-feeZat, r[0]['amountZat']) assert_true(r[0]['change'], "Note valued at (0.4-fee) should be change") assert_equal(no_memo, r[0]['memo']) # The old note still exists (it's immutable), even though it is spent assert_equal(Decimal('1.0'), r[1]['amount']) + assert_equal(100000000, r[1]['amountZat']) assert_false(r[1]['change'], "Note valued at 1.0 should not be change") assert_equal(my_memo, r[1]['memo']) @@ -217,6 +221,7 @@ class ListReceivedTest (BitcoinTestFramework): assert_equal(1, len(r), "zaddr2 Should have received 1 notes") assert_equal(txid, r[0]['txid']) assert_equal(Decimal('0.6'), r[0]['amount']) + assert_equal(60000000, r[0]['amountZat']) assert_false(r[0]['change'], "Note valued at 0.6 should not be change") assert_equal(no_memo, r[0]['memo']) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index cf43d0e21..d2a9aa28d 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3386,6 +3386,7 @@ UniValue z_listreceivedbyaddress(const UniValue& params, bool fHelp) "{\n" " \"txid\": \"txid\", (string) the transaction id\n" " \"amount\": xxxxx, (numeric) the amount of value in the note\n" + " \"amountZat\" : xxxx (numeric) The amount in zatoshis\n" " \"memo\": xxxxx, (string) hexadecimal string representation of memo field\n" " \"confirmations\" : n, (numeric) the number of confirmations\n" " \"blockheight\": n, (numeric) The block height containing the transaction\n" @@ -3440,6 +3441,7 @@ UniValue z_listreceivedbyaddress(const UniValue& params, bool fHelp) UniValue obj(UniValue::VOBJ); obj.pushKV("txid", entry.jsop.hash.ToString()); obj.pushKV("amount", ValueFromAmount(CAmount(entry.note.value()))); + obj.pushKV("amountZat", CAmount(entry.note.value())); std::string data(entry.memo.begin(), entry.memo.end()); obj.pushKV("memo", HexStr(data)); obj.pushKV("jsindex", entry.jsop.js); @@ -3461,6 +3463,7 @@ UniValue z_listreceivedbyaddress(const UniValue& params, bool fHelp) UniValue obj(UniValue::VOBJ); obj.pushKV("txid", entry.op.hash.ToString()); obj.pushKV("amount", ValueFromAmount(CAmount(entry.note.value()))); + obj.pushKV("amountZat", CAmount(entry.note.value())); obj.pushKV("memo", HexStr(entry.memo)); obj.pushKV("outindex", (int)entry.op.n); obj.pushKV("confirmations", entry.confirmations); From 25bccb4814a01d88dfd7ff17cc1898aa376f437a Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 22 Apr 2020 09:13:15 -0300 Subject: [PATCH 025/725] replace with AssertionError assert_equal in receivedby.py Co-Authored-By: Daira Hopwood --- qa/rpc-tests/receivedby.py | 34 ++++++++++++++-------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/qa/rpc-tests/receivedby.py b/qa/rpc-tests/receivedby.py index 2c23cb892..6e831dbe7 100755 --- a/qa/rpc-tests/receivedby.py +++ b/qa/rpc-tests/receivedby.py @@ -6,6 +6,7 @@ # Exercise the listreceivedbyaddress API from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import assert_equal from decimal import Decimal @@ -95,25 +96,22 @@ class ReceivedByTest(BitcoinTestFramework): # Check balance is 0 because of 0 confirmations balance = self.nodes[1].getreceivedbyaddress(addr) - if balance != Decimal("0.0"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.0"), "Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + # Check balance is 0.1 balance = self.nodes[1].getreceivedbyaddress(addr,0) - if balance != Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.1"), "Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) # Bury Tx under 10 block so it will be returned by the default getreceivedbyaddress self.nodes[1].generate(10) self.sync_all() balance = self.nodes[1].getreceivedbyaddress(addr) - if balance != Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.1"), "Wrong balance returned by getreceivedbyaddress, %0.2f"%(balance)) # Get balance as integer balance = self.nodes[1].getreceivedbyaddress(addr, 1, True) - if balance != int("10000000"): - raise AssertionError("Wrong balance returned by getreceivedbyaddress, %i"%(balance)) + assert_equal(balance, 10000000, "Wrong balance returned by getreceivedbyaddress, %i"%(balance)) ''' listreceivedbyaccount + getreceivedbyaccount Test @@ -122,6 +120,7 @@ class ReceivedByTest(BitcoinTestFramework): addrArr = self.nodes[1].getnewaddress() account = self.nodes[1].getaccount(addrArr) received_by_account_json = get_sub_array_from_array(self.nodes[1].listreceivedbyaccount(),{"account":account}) + if len(received_by_account_json) == 0: raise AssertionError("No accounts found in node") balance_by_account = self.nodes[1].getreceivedbyaccount(account) @@ -136,25 +135,22 @@ class ReceivedByTest(BitcoinTestFramework): # getreceivedbyaccount should return same balance because of 0 confirmations balance = self.nodes[1].getreceivedbyaccount(account) - if balance != balance_by_account: - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, balance_by_account, "Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) self.nodes[1].generate(10) self.sync_all() # listreceivedbyaccount should return updated account balance check_array_result(self.nodes[1].listreceivedbyaccount(), {"account":account}, - {"account":received_by_account_json["account"], "amount":(received_by_account_json["amount"] + Decimal("0.1")), "amountZat":int("30000000")}) + {"account":received_by_account_json["account"], "amount":(received_by_account_json["amount"] + Decimal("0.1")), "amountZat":30000000}) # getreceivedbyaccount should return updates balance balance = self.nodes[1].getreceivedbyaccount(account) - if balance != balance_by_account + Decimal("0.1"): - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, balance_by_account + Decimal("0.1"), "Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) # Get balance as integer balance = self.nodes[1].getreceivedbyaccount(account, 1, True) - if balance != int("30000000"): - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %i"%(balance)) + assert_equal(balance, 30000000, "Wrong balance returned by getreceivedbyaccount, %i"%(balance)) # Create a new account named "mynewaccount" that has a 0 balance self.nodes[1].getaccountaddress("mynewaccount") @@ -162,14 +158,12 @@ class ReceivedByTest(BitcoinTestFramework): if len(received_by_account_json) == 0: raise AssertionError("No accounts found in node") - # Test includeempty of listreceivedbyaccount - if received_by_account_json["amount"] != Decimal("0.0"): - raise AssertionError("Wrong balance returned by listreceivedbyaccount, %0.2f"%(received_by_account_json["amount"])) + # Test listreceivedbyaccount for 0 amount accounts + assert_equal(received_by_account_json["amount"], Decimal("0.0"), "Wrong balance returned by listreceivedbyaccount, %0.2f"%(received_by_account_json["amount"])) # Test getreceivedbyaccount for 0 amount accounts balance = self.nodes[1].getreceivedbyaccount("mynewaccount") - if balance != Decimal("0.0"): - raise AssertionError("Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) + assert_equal(balance, Decimal("0.0"), "Wrong balance returned by getreceivedbyaccount, %0.2f"%(balance)) if __name__ == '__main__': ReceivedByTest().main() From 24d3516e35dc3f06db4eaa5c081192fac1892725 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 22 Apr 2020 09:23:22 -0300 Subject: [PATCH 026/725] Fix casting in wallet.py Co-Authored-By: Daira Hopwood --- qa/rpc-tests/wallet.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index 5803c117d..a1f08ffba 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -264,12 +264,12 @@ class WalletTest (BitcoinTestFramework): self.nodes[0].generate(1) sync_blocks(self.nodes) - # tx should be added to balance because after restarting the nodes tx should be broadcastet + # tx should be added to balance because after restarting the nodes tx should be broadcast assert_equal(self.nodes[2].getbalance(), Decimal('13.99800000')) #should not be assert_equal(self.nodes[2].getbalance("*"), Decimal('13.99800000')) #should not be # check integer balances from getbalance - assert_equal(self.nodes[2].getbalance("*", 1, False, True), int('1399800000')) #should not be + assert_equal(self.nodes[2].getbalance("*", 1, False, True), 1399800000) #should not be # send from node 0 to node 2 taddr mytaddr = self.nodes[2].getnewaddress() @@ -282,7 +282,7 @@ class WalletTest (BitcoinTestFramework): assert_equal(mybalance, Decimal('10.0')) # check integer balances from z_getbalance - assert_equal(self.nodes[2].z_getbalance(mytaddr, 1, True), int('10')) + assert_equal(self.nodes[2].z_getbalance(mytaddr, 1, True), 10) mytxdetails = self.nodes[2].gettransaction(mytxid) myvjoinsplits = mytxdetails["vjoinsplit"] @@ -386,18 +386,18 @@ class WalletTest (BitcoinTestFramework): txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "2") txObj = self.nodes[0].gettransaction(txId) assert_equal(txObj['amount'], Decimal('-2.00000000')) - assert_equal(txObj['amountZat'], int('-200000000')) + assert_equal(txObj['amountZat'], -200000000) txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "0.0001") txObj = self.nodes[0].gettransaction(txId) assert_equal(txObj['amount'], Decimal('-0.00010000')) - assert_equal(txObj['amountZat'], int('-10000')) + assert_equal(txObj['amountZat'], -10000) #check if JSON parser can handle scientific notation in strings txId = self.nodes[0].sendtoaddress(self.nodes[2].getnewaddress(), "1e-4") txObj = self.nodes[0].gettransaction(txId) assert_equal(txObj['amount'], Decimal('-0.00010000')) - assert_equal(txObj['amountZat'], int('-10000')) + assert_equal(txObj['amountZat'], -10000) #this should fail errorString = "" From 507e1623f9efeea90031f31cdb1bf250852e0435 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 22 Apr 2020 11:00:32 -0300 Subject: [PATCH 027/725] simplify inzat balances logic Co-Authored-By: Daira Hopwood --- src/wallet/rpcwallet.cpp | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index d2a9aa28d..851541a12 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -650,9 +650,9 @@ UniValue getreceivedbyaddress(const UniValue& params, bool fHelp) } // inZat - if (params.size() > 2) - if (params[2].get_bool()) - return nAmount; + if (params.size() > 2 && params[2].get_bool()) { + return nAmount; + } return ValueFromAmount(nAmount); } @@ -713,9 +713,9 @@ UniValue getreceivedbyaccount(const UniValue& params, bool fHelp) } // inZat - if (params.size() > 2) - if (params[2].get_bool()) - return nAmount; + if (params.size() > 2 && params[2].get_bool()) { + return nAmount; + } return ValueFromAmount(nAmount); } @@ -818,9 +818,9 @@ UniValue getbalance(const UniValue& params, bool fHelp) } // inZat - if (params.size() > 3) - if (params[3].get_bool()) - return nBalance; + if (params.size() > 3 && params[3].get_bool()) { + return nBalance; + } return ValueFromAmount(nBalance); } @@ -1685,7 +1685,7 @@ UniValue listsinceblock(const UniValue& params, bool fHelp) " or 'expired'. Available for 'send' and 'receive' category of transactions.\n" " \"amount\": x.xxx, (numeric) The amount in " + CURRENCY_UNIT + ". This is negative for the 'send' category, and for the 'move' category for moves \n" " outbound. It is positive for the 'receive' category, and for the 'move' category for inbound funds.\n" - " \"amountZat\": x.xxx, (numeric) The amount in zatoshis. Negative and positive are the same as 'amount' field.\n" + " \"amountZat\": x.xxx, (numeric) The amount in zatoshis. Negative and positive are the same as for the 'amount' field.\n" " \"vout\" : n, (numeric) the vout value\n" " \"fee\": x.xxx, (numeric) The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the 'send' category of transactions.\n" " \"confirmations\": n, (numeric) The number of confirmations for the transaction. Available for 'send' and 'receive' category of transactions.\n" @@ -3541,9 +3541,9 @@ UniValue z_getbalance(const UniValue& params, bool fHelp) } // inZat - if (params.size() > 3) - if (params[3].get_bool()) - return nBalance; + if (params.size() > 3 && params[3].get_bool()) { + return nBalance; + } return ValueFromAmount(nBalance); } From 26e4889f874e6728ff2afe66529ef0b8da82ebf3 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 22 Apr 2020 11:04:16 -0300 Subject: [PATCH 028/725] Fix casting in listtransactions.py --- qa/rpc-tests/listtransactions.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/qa/rpc-tests/listtransactions.py b/qa/rpc-tests/listtransactions.py index 5cdc48d04..990f9a605 100755 --- a/qa/rpc-tests/listtransactions.py +++ b/qa/rpc-tests/listtransactions.py @@ -39,29 +39,29 @@ class ListTransactionsTest(BitcoinTestFramework): self.sync_all() check_array_result(self.nodes[0].listtransactions(), {"txid":txid}, - {"category":"send","account":"","amount":Decimal("-0.1"),"amountZat":int("-10000000"),"confirmations":0}) + {"category":"send","account":"","amount":Decimal("-0.1"),"amountZat":-10000000,"confirmations":0}) check_array_result(self.nodes[1].listtransactions(), {"txid":txid}, - {"category":"receive","account":"","amount":Decimal("0.1"),"amountZat":int("10000000"),"confirmations":0}) + {"category":"receive","account":"","amount":Decimal("0.1"),"amountZat":10000000,"confirmations":0}) # mine a block, confirmations should change: self.nodes[0].generate(1) self.sync_all() check_array_result(self.nodes[0].listtransactions(), {"txid":txid}, - {"category":"send","account":"","amount":Decimal("-0.1"),"amountZat":int("-10000000"),"confirmations":1}) + {"category":"send","account":"","amount":Decimal("-0.1"),"amountZat":-10000000,"confirmations":1}) check_array_result(self.nodes[1].listtransactions(), {"txid":txid}, - {"category":"receive","account":"","amount":Decimal("0.1"),"amountZat":int("10000000"),"confirmations":1}) + {"category":"receive","account":"","amount":Decimal("0.1"),"amountZat":10000000,"confirmations":1}) # send-to-self: txid = self.nodes[0].sendtoaddress(self.nodes[0].getnewaddress(), 0.2) check_array_result(self.nodes[0].listtransactions(), {"txid":txid, "category":"send"}, - {"amount":Decimal("-0.2"),"amountZat":int("-20000000")}) + {"amount":Decimal("-0.2"),"amountZat":-20000000}) check_array_result(self.nodes[0].listtransactions(), {"txid":txid, "category":"receive"}, - {"amount":Decimal("0.2"),"amountZat":int("20000000")}) + {"amount":Decimal("0.2"),"amountZat":20000000}) # sendmany from node1: twice to self, twice to node2: send_to = { self.nodes[0].getnewaddress() : 0.11, @@ -71,28 +71,28 @@ class ListTransactionsTest(BitcoinTestFramework): txid = self.nodes[1].sendmany("", send_to) self.sync_all() check_array_result(self.nodes[1].listtransactions(), - {"category":"send","amount":Decimal("-0.11"),"amountZat":int("-11000000")}, + {"category":"send","amount":Decimal("-0.11"),"amountZat":-11000000}, {"txid":txid} ) check_array_result(self.nodes[0].listtransactions(), - {"category":"receive","amount":Decimal("0.11"),"amountZat":int("11000000")}, + {"category":"receive","amount":Decimal("0.11"),"amountZat":11000000}, {"txid":txid} ) check_array_result(self.nodes[1].listtransactions(), - {"category":"send","amount":Decimal("-0.22"),"amountZat":int("-22000000")}, + {"category":"send","amount":Decimal("-0.22"),"amountZat":-22000000}, {"txid":txid} ) check_array_result(self.nodes[1].listtransactions(), - {"category":"receive","amount":Decimal("0.22"),"amountZat":int("22000000")}, + {"category":"receive","amount":Decimal("0.22"),"amountZat":22000000}, {"txid":txid} ) check_array_result(self.nodes[1].listtransactions(), - {"category":"send","amount":Decimal("-0.33"),"amountZat":int("-33000000")}, + {"category":"send","amount":Decimal("-0.33"),"amountZat":-33000000}, {"txid":txid} ) check_array_result(self.nodes[0].listtransactions(), - {"category":"receive","amount":Decimal("0.33"),"amountZat":int("33000000")}, + {"category":"receive","amount":Decimal("0.33"),"amountZat":33000000}, {"txid":txid, "account" : ""} ) check_array_result(self.nodes[1].listtransactions(), - {"category":"send","amount":Decimal("-0.44"),"amountZat":int("-44000000")}, + {"category":"send","amount":Decimal("-0.44"),"amountZat":-44000000}, {"txid":txid, "account" : ""} ) check_array_result(self.nodes[1].listtransactions(), - {"category":"receive","amount":Decimal("0.44"),"amountZat":int("44000000")}, + {"category":"receive","amount":Decimal("0.44"),"amountZat":44000000}, {"txid":txid, "account" : ""} ) multisig = self.nodes[1].createmultisig(1, [self.nodes[1].getnewaddress()]) @@ -102,7 +102,7 @@ class ListTransactionsTest(BitcoinTestFramework): self.sync_all() assert(len(self.nodes[0].listtransactions("watchonly", 100, 0, False)) == 0) check_array_result(self.nodes[0].listtransactions("watchonly", 100, 0, True), - {"category":"receive","amount":Decimal("0.1"),"amountZat":int("10000000")}, + {"category":"receive","amount":Decimal("0.1"),"amountZat":10000000}, {"txid":txid, "account" : "watchonly"} ) if __name__ == '__main__': From 1805466d5c555778017bdb165688fd699bcbcdbb Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 22 Apr 2020 11:21:12 -0300 Subject: [PATCH 029/725] add MINOR_CURRENCY_UNIT --- src/amount.cpp | 1 + src/amount.h | 1 + src/wallet/rpcwallet.cpp | 16 ++++++++-------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/amount.cpp b/src/amount.cpp index 3572e59c8..78e90bc78 100644 --- a/src/amount.cpp +++ b/src/amount.cpp @@ -8,6 +8,7 @@ #include "tinyformat.h" const std::string CURRENCY_UNIT = "ZEC"; +const std::string MINOR_CURRENCY_UNIT = "zatoshis"; CFeeRate::CFeeRate(const CAmount& nFeePaid, size_t nSize) { diff --git a/src/amount.h b/src/amount.h index 9913e565e..73f7741c0 100644 --- a/src/amount.h +++ b/src/amount.h @@ -17,6 +17,7 @@ static const CAmount COIN = 100000000; static const CAmount CENT = 1000000; extern const std::string CURRENCY_UNIT; +extern const std::string MINOR_CURRENCY_UNIT; /** No amount larger than this (in satoshi) is valid. * diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 851541a12..92a36bdb4 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -1304,7 +1304,7 @@ UniValue listreceivedbyaddress(const UniValue& params, bool fHelp) " \"address\" : \"receivingaddress\", (string) The receiving address\n" " \"account\" : \"accountname\", (string) DEPRECATED. The account of the receiving address. The default account is \"\".\n" " \"amount\" : x.xxx, (numeric) The total amount in " + CURRENCY_UNIT + " received by the address\n" - " \"amountZat\" : xxxx (numeric) The amount in zatoshis\n" + " \"amountZat\" : xxxx (numeric) The amount in " + MINOR_CURRENCY_UNIT + "\n" " \"confirmations\" : n (numeric) The number of confirmations of the most recent transaction included\n" " }\n" " ,...\n" @@ -1341,7 +1341,7 @@ UniValue listreceivedbyaccount(const UniValue& params, bool fHelp) " \"involvesWatchonly\" : true, (bool) Only returned if imported addresses were involved in transaction\n" " \"account\" : \"accountname\", (string) The account name of the receiving account\n" " \"amount\" : x.xxx, (numeric) The total amount received by addresses with this account\n" - " \"amountZat\" : xxxx (numeric) The amount in zatoshis\n" + " \"amountZat\" : xxxx (numeric) The amount in " + MINOR_CURRENCY_UNIT + "\n" " \"confirmations\" : n (numeric) The number of confirmations of the most recent transaction included\n" " }\n" " ,...\n" @@ -1487,7 +1487,7 @@ UniValue listtransactions(const UniValue& params, bool fHelp) " \"amount\": x.xxx, (numeric) The amount in " + CURRENCY_UNIT + ". This is negative for the 'send' category, and for the\n" " 'move' category for moves outbound. It is positive for the 'receive' category,\n" " and for the 'move' category for inbound funds.\n" - " \"amountZat\": x.xxx, (numeric) The amount in zatoshis. Negative and positive are the same as 'amount' field.\n" + " \"amountZat\": x.xxx, (numeric) The amount in " + MINOR_CURRENCY_UNIT + ". Negative and positive are the same as 'amount' field.\n" " \"vout\" : n, (numeric) the vout value\n" " \"fee\": x.xxx, (numeric) The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the \n" " 'send' category of transactions.\n" @@ -1685,7 +1685,7 @@ UniValue listsinceblock(const UniValue& params, bool fHelp) " or 'expired'. Available for 'send' and 'receive' category of transactions.\n" " \"amount\": x.xxx, (numeric) The amount in " + CURRENCY_UNIT + ". This is negative for the 'send' category, and for the 'move' category for moves \n" " outbound. It is positive for the 'receive' category, and for the 'move' category for inbound funds.\n" - " \"amountZat\": x.xxx, (numeric) The amount in zatoshis. Negative and positive are the same as for the 'amount' field.\n" + " \"amountZat\": x.xxx, (numeric) The amount in " + MINOR_CURRENCY_UNIT + ". Negative and positive are the same as for the 'amount' field.\n" " \"vout\" : n, (numeric) the vout value\n" " \"fee\": x.xxx, (numeric) The amount of the fee in " + CURRENCY_UNIT + ". This is negative and only available for the 'send' category of transactions.\n" " \"confirmations\": n, (numeric) The number of confirmations for the transaction. Available for 'send' and 'receive' category of transactions.\n" @@ -1772,7 +1772,7 @@ UniValue gettransaction(const UniValue& params, bool fHelp) "{\n" " \"status\" : \"mined|waiting|expiringsoon|expired\", (string) The transaction status, can be 'mined', 'waiting', 'expiringsoon' or 'expired'\n" " \"amount\" : x.xxx, (numeric) The transaction amount in " + CURRENCY_UNIT + "\n" - " \"amountZat\" : x (numeric) The amount in zatoshis\n" + " \"amountZat\" : x (numeric) The amount in " + MINOR_CURRENCY_UNIT + "\n" " \"confirmations\" : n, (numeric) The number of confirmations\n" " \"blockhash\" : \"hash\", (string) The block hash\n" " \"blockindex\" : xx, (numeric) The block index\n" @@ -1786,7 +1786,7 @@ UniValue gettransaction(const UniValue& params, bool fHelp) " \"address\" : \"zcashaddress\", (string) The Zcash address involved in the transaction\n" " \"category\" : \"send|receive\", (string) The category, either 'send' or 'receive'\n" " \"amount\" : x.xxx (numeric) The amount in " + CURRENCY_UNIT + "\n" - " \"amountZat\" : x (numeric) The amount in zatoshis\n" + " \"amountZat\" : x (numeric) The amount in " + MINOR_CURRENCY_UNIT + "\n" " \"vout\" : n, (numeric) the vout value\n" " }\n" " ,...\n" @@ -2411,7 +2411,7 @@ UniValue listunspent(const UniValue& params, bool fHelp) " \"account\" : \"account\", (string) DEPRECATED. The associated account, or \"\" for the default account\n" " \"scriptPubKey\" : \"key\", (string) the script key\n" " \"amount\" : x.xxx, (numeric) the transaction amount in " + CURRENCY_UNIT + "\n" - " \"amountZat\" : xxxx (numeric) the transaction amount in zatoshis\n" + " \"amountZat\" : xxxx (numeric) the transaction amount in " + MINOR_CURRENCY_UNIT + "\n" " \"confirmations\" : n, (numeric) The number of confirmations\n" " \"redeemScript\" : n (string) The redeemScript if scriptPubKey is P2SH\n" " \"spendable\" : xxx (bool) Whether we have the private keys to spend this output\n" @@ -3386,7 +3386,7 @@ UniValue z_listreceivedbyaddress(const UniValue& params, bool fHelp) "{\n" " \"txid\": \"txid\", (string) the transaction id\n" " \"amount\": xxxxx, (numeric) the amount of value in the note\n" - " \"amountZat\" : xxxx (numeric) The amount in zatoshis\n" + " \"amountZat\" : xxxx (numeric) The amount in " + MINOR_CURRENCY_UNIT + "\n" " \"memo\": xxxxx, (string) hexadecimal string representation of memo field\n" " \"confirmations\" : n, (numeric) the number of confirmations\n" " \"blockheight\": n, (numeric) The block height containing the transaction\n" From d42e0a433cafee963a9cf79ecd33a1165a82e89c Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 18 May 2020 18:02:22 -0300 Subject: [PATCH 030/725] remove additional not needed casts from py tests --- qa/rpc-tests/receivedby.py | 4 ++-- qa/rpc-tests/wallet.py | 2 +- qa/rpc-tests/wallet_listreceived.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/qa/rpc-tests/receivedby.py b/qa/rpc-tests/receivedby.py index 6e831dbe7..f60bdfa97 100755 --- a/qa/rpc-tests/receivedby.py +++ b/qa/rpc-tests/receivedby.py @@ -72,11 +72,11 @@ class ReceivedByTest(BitcoinTestFramework): self.sync_all() check_array_result(self.nodes[1].listreceivedbyaddress(), {"address":addr}, - {"address":addr, "account":"", "amount":Decimal("0.1"), "amountZat":int("10000000"), "confirmations":10, "txids":[txid,]}) + {"address":addr, "account":"", "amount":Decimal("0.1"), "amountZat":10000000, "confirmations":10, "txids":[txid,]}) # With min confidence < 10 check_array_result(self.nodes[1].listreceivedbyaddress(5), {"address":addr}, - {"address":addr, "account":"", "amount":Decimal("0.1"), "amountZat":int("10000000"), "confirmations":10, "txids":[txid,]}) + {"address":addr, "account":"", "amount":Decimal("0.1"), "amountZat":10000000, "confirmations":10, "txids":[txid,]}) # With min confidence > 10, should not find Tx check_array_result(self.nodes[1].listreceivedbyaddress(11),{"address":addr},{ },True) diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index a1f08ffba..626818a3b 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -220,7 +220,7 @@ class WalletTest (BitcoinTestFramework): if uTx['txid'] == zeroValueTxid: found = True assert_equal(uTx['amount'], Decimal('0.00000000')) - assert_equal(uTx['amountZat'], int('0')) + assert_equal(uTx['amountZat'], 0) assert(found) #do some -walletbroadcast tests diff --git a/qa/rpc-tests/wallet_listreceived.py b/qa/rpc-tests/wallet_listreceived.py index ad30e9537..7addb4aae 100755 --- a/qa/rpc-tests/wallet_listreceived.py +++ b/qa/rpc-tests/wallet_listreceived.py @@ -15,7 +15,7 @@ my_memo = my_memo + '0'*(1024-len(my_memo)) no_memo = 'f6' + ('0'*1022) # see section 5.5 of the protocol spec fee = Decimal('0.0001') -feeZat = Decimal('10000') +feeZat = 10000 class ListReceivedTest (BitcoinTestFramework): @@ -205,7 +205,7 @@ class ListReceivedTest (BitcoinTestFramework): assert_equal(txid, r[0]['txid']) assert_equal(Decimal('0.4')-fee, r[0]['amount']) - assert_equal(Decimal('40000000')-feeZat, r[0]['amountZat']) + assert_equal(40000000-feeZat, r[0]['amountZat']) assert_true(r[0]['change'], "Note valued at (0.4-fee) should be change") assert_equal(no_memo, r[0]['memo']) From 6470237c09ff10a08625f82bfe3d5376d427557a Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 21 Mar 2019 15:13:19 +1300 Subject: [PATCH 031/725] qa: Smoke test driver --- qa/zcash/smoke_tests.py | 186 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100755 qa/zcash/smoke_tests.py diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py new file mode 100755 index 000000000..3e479eab4 --- /dev/null +++ b/qa/zcash/smoke_tests.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# +# Execute the standard smoke tests for Zcash releases. +# + +import argparse +import sys + +# +# Smoke test definitions +# + +# (case, expected_mainnet, expected_testnet) +SMOKE_TESTS = [ + # zcashd start/stop/restart flows + ('1a', True, True), # zcashd start + ('1b', True, True), # Graceful zcashd stop + ('1c', True, True), # Ungraceful zcashd stop + ('1d', True, True), # zcashd start; graceful zcashd stop; zcashd start + ('1e', True, True), # zcashd start; ungraceful zcashd stop; zcashd start + # Control + ('2a', True, True), # Run getinfo + ('2b', True, True), # Run help + # Address generation + ('3a', True, True), # Generate a Sprout z-addr + ('3b', True, True), # Generate multiple Sprout z-addrs + ('3c', True, True), # Generate a t-addr + ('3d', True, True), # Generate multiple t-addrs + ('3e', True, True), # Generate a Sapling z-addr + ('3f', True, True), # Generate multiple Sapling z-addrs + # Transactions + ('4a', True, True ), # Send funds from Sprout z-addr to same Sprout z-addr + ('4b', True, True ), # Send funds from Sprout z-addr to a different Sprout z-addr + ('4c', True, True ), # Send funds from Sprout z-addr to a t-addr + ('4d', True, True ), # Send funds from t-addr to Sprout z-addr + ('4e', True, True ), # Send funds from t-addr to t-addr + ('4f', True, True ), # Send funds from t-addr to Sapling z-addr + ('4g', True, True ), # Send funds from Sapling z-addr to same Sapling z-addr + ('4h', True, True ), # Send funds from Sapling z-addr to a different Sapling z-addr + ('4i', True, True ), # Send funds from Sapling z-addr to a t-addr + ('4j', False, False), # Send funds from Sprout z-addr to Sapling z-addr + ('4k', True, True ), # Send funds from Sprout z-addr to multiple Sprout z-addrs + ('4l', True, True ), # Send funds from Sprout z-addr to multiple t-addrs + ('4m', True, True ), # Send funds from Sprout z-addr to t-addr and Sprout z-addrs + ('4n', False, False), # Send funds from Sprout z-addr to t-addr and Sapling z-addr + ('4o', False, False), # Send funds from Sprout z-addr to multiple Sapling z-addrs + ('4p', True, True ), # Send funds from t-addr to multiple t-addrs + ('4q', True, True ), # Send funds from t-addr to multiple Sprout z-addrs + ('4r', True, True ), # Send funds from t-addr to multiple Sapling z-addrs + ('4s', False, False), # Send funds from t-addr to Sprout z-addr and Sapling z-addr + ('4t', True, True ), # Send funds from Sapling z-addr to multiple Sapling z-addrs + ('4u', False, False), # Send funds from Sapling z-addr to multiple Sprout z-addrs + ('4v', True, True ), # Send funds from Sapling z-addr to multiple t-addrs + ('4w', True, True ), # Send funds from Sapling z-addr to t-addr and Sapling z-addr + ('4x', False, False), # Send funds from Sapling z-addr to Sapling z-addr and Sprout z-addr + ('4y', True, True ), # Send funds from t-addr to Sprout z-addr using z_mergetoaddress + ('4z', True, True ), # Send funds from 2 different t-addrs to Sprout z-addr using z_mergetoaddress + ('4aa', False, False), # Send funds from the same 2 t-addrs to Sprout z-addr using z_mergetoaddress + ('4bb', True, True ), # Send funds from 2 different t-addrs to Sapling z-addr using z_mergetoaddress + ('4cc', True, True ), # Send funds from t-addr to Sapling z-addr using z_mergetoaddress + ('4dd', True, True ), # Send funds from t-addr and Sprout z-addr to Sprout z-addr using z_mergetoaddress + ('4ee', True, True ), # Send funds from t-addr and Sapling z-addr to Sapling z-addr using z_mergetoaddress + ('4ff', True, True ), # Send funds from Sprout z-addr and Sprout z-addr to Sprout z-addr using z_mergetoaddress + ('4gg', True, True ), # Send funds from Sapling z-addr and Sapling z-addr to Sapling z-addr using z_mergetoaddress + # Wallet + ('5a', True, True), # After generating multiple z-addrs, run z_listaddresses + ('5b', True, True), # Run z_validateaddress with a Sprout z-addr + ('5c', True, True), # Run z_validateaddress with a Sapling z-addr + ('5d', True, True), # After a transaction, run z_listunspent + ('5e', True, True), # After a transaction, run z_listreceivedbyaddress + ('5f', True, True), # After a transaction, run z_getbalance + ('5g', True, True), # After a transaction, run z_gettotalbalance + ('5h', True, True), # Run z_exportkey using a Sprout z-addr + ('5i', True, True), # Run z_importkey using the zkey from a Sprout z-addr + ('5j', True, True), # Run z_exportkey using a Sapling z-addr + ('5k', True, True), # Run z_importkey using the zkey from a Sapling z-addr + ('5l', True, True), # Run z_exportwallet + ('5m', True, True), # Run z_importwallet + ('5n', True, True), # Run z_shieldcoinbase + ('5o', True, True), # Run getwalletinfo + # Network + ('6a', True, True), # Run getpeerinfo + ('6b', True, True), # Run getnetworkinfo + ('6c', True, False), # Run getdeprecationinfo + ('6d', True, True), # Run getconnectioncount + ('6e', True, True), # Run getaddednodeinfo + # Mining + ('7a', True, True), # Run getblocksubsidy + ('7b', True, True), # Run getblocktemplate + ('7c', True, True), # Run getmininginfo + ('7d', True, True), # Run getnetworkhashps + ('7e', True, True), # Run getnetworksolps +] + + +# +# Test stages +# + +STAGES = [ +] + +STAGE_COMMANDS = { +} + +def run_stage(stage): + print('Running stage %s' % stage) + print('=' * (len(stage) + 14)) + print() + + cmd = STAGE_COMMANDS[stage] + if cmd is not None: + ret = cmd() + else: + print('WARNING: stage not yet implemented, skipping') + ret = {} + + print() + print('-' * (len(stage) + 15)) + print('Finished stage %s' % stage) + print() + + return ret + + +# +# Test driver +# + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--list-stages', dest='list', action='store_true') + parser.add_argument('--mainnet', action='store_true', help='Use mainnet instead of testnet') + parser.add_argument('stage', nargs='*', default=STAGES, + help='One of %s'%STAGES) + args = parser.parse_args() + + # Check for list + if args.list: + for s in STAGES: + print(s) + sys.exit(0) + + # Check validity of stages + for s in args.stage: + if s not in STAGES: + print("Invalid stage '%s' (choose from %s)" % (s, STAGES)) + sys.exit(1) + + # Run the stages + results = {} + for s in args.stage: + results.update(run_stage(s)) + + passed = True + print() + print('========================') + print(' Results') + print('========================') + print('Case | Expected | Actual') + print('========================') + for test_case in SMOKE_TESTS: + case = test_case[0] + expected = test_case[1 if args.mainnet else 2] + if case in results: + actual = results[case] + actual_str = '%s%s' % ( + 'Passed' if actual else 'Failed', + '' if expected == actual else '!!!' + ) + passed &= (expected == actual) + else: + actual_str = ' N/A' + print('%s | %s | %s' % ( + case.ljust(4), + ' Passed ' if expected else ' Failed ', + actual_str + )) + + if not passed: + print() + print("!!! One or more smoke test stages failed !!!") + sys.exit(1) + +if __name__ == '__main__': + main() From 6d1563d7a3cccebc947a1e5d260299b9693f1002 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 21 Mar 2019 22:59:58 +1300 Subject: [PATCH 032/725] qa: Run Zcash node for smoke tests --- qa/zcash/smoke_tests.py | 121 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 117 insertions(+), 4 deletions(-) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index 3e479eab4..0f58f1e25 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -3,8 +3,22 @@ # Execute the standard smoke tests for Zcash releases. # -import argparse +# Add RPC test_framework to module search path: +import os import sys +sys.path.insert( + 0, + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'rpc-tests', + 'test_framework', + ) +) + +import argparse +import subprocess + +from authproxy import AuthServiceProxy, JSONRPCException # # Smoke test definitions @@ -103,14 +117,14 @@ STAGES = [ STAGE_COMMANDS = { } -def run_stage(stage): +def run_stage(stage, zcash): print('Running stage %s' % stage) print('=' * (len(stage) + 14)) print() cmd = STAGE_COMMANDS[stage] if cmd is not None: - ret = cmd() + ret = cmd(zcash) else: print('WARNING: stage not yet implemented, skipping') ret = {} @@ -123,6 +137,93 @@ def run_stage(stage): return ret +# +# Zcash wrapper +# + +class ZcashNode(object): + def __init__(self, datadir, wallet, zcashd=None, zcash_cli=None): + if zcashd is None: + zcashd = os.getenv('ZCASHD', 'zcashd') + if zcash_cli is None: + zcash_cli = os.getenv('ZCASHCLI', 'zcash-cli') + + self.__datadir = datadir + self.__wallet = wallet + self.__zcashd = zcashd + self.__zcash_cli = zcash_cli + self.__process = None + self.__proxy = None + + def start(self, testnet=True, extra_args=None, timewait=None): + if self.__proxy is not None: + raise RuntimeError('Already started') + + rpcuser = 'st' + rpcpassword = 'st' + + args = [ + self.__zcashd, + '-datadir=%s' % self.__datadir, + '-wallet=%s' % self.__wallet, + '-rpcuser=%s' % rpcuser, + '-rpcpassword=%s' % rpcpassword, + '-showmetrics=0', + '-experimentalfeatures', + '-zmergetoaddress', + ] + if testnet: + args.append('-testnet=1') + if extra_args is not None: + args.extend(extra_args) + + self.__process = subprocess.Popen(args) + + cli_args = [ + self.__zcash_cli, + '-datadir=%s' % self.__datadir, + '-rpcuser=%s' % rpcuser, + '-rpcpassword=%s' % rpcpassword, + '-rpcwait', + ] + if testnet: + cli_args.append('-testnet=1') + cli_args.append('getblockcount') + + devnull = open('/dev/null', 'w+') + if os.getenv('PYTHON_DEBUG', ''): + print('start_node: zcashd started, calling zcash-cli -rpcwait getblockcount') + subprocess.check_call(cli_args, stdout=devnull) + if os.getenv('PYTHON_DEBUG', ''): + print('start_node: calling zcash-cli -rpcwait getblockcount returned') + devnull.close() + + rpcuserpass = '%s:%s' % (rpcuser, rpcpassword) + rpchost = '127.0.0.1' + rpcport = 18232 if testnet else 8232 + + url = 'http://%s@%s:%d' % (rpcuserpass, rpchost, rpcport) + if timewait is not None: + self.__proxy = AuthServiceProxy(url, timeout=timewait) + else: + self.__proxy = AuthServiceProxy(url) + + def stop(self): + if self.__proxy is None: + raise RuntimeError('Not running') + + self.__proxy.stop() + self.__process.wait() + self.__proxy = None + self.__process = None + + def __getattr__(self, name): + if self.__proxy is None: + raise RuntimeError('Not running') + + return self.__proxy.__getattr__(name) + + # # Test driver # @@ -131,6 +232,8 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--list-stages', dest='list', action='store_true') parser.add_argument('--mainnet', action='store_true', help='Use mainnet instead of testnet') + parser.add_argument('--wallet', default='wallet.dat', help='Wallet file to use (within data directory)') + parser.add_argument('datadir', help='Data directory to use for smoke testing', default=None) parser.add_argument('stage', nargs='*', default=STAGES, help='One of %s'%STAGES) args = parser.parse_args() @@ -147,10 +250,20 @@ def main(): print("Invalid stage '%s' (choose from %s)" % (s, STAGES)) sys.exit(1) + # Start zcashd + zcash = ZcashNode(args.datadir, args.wallet) + print('Starting zcashd...') + zcash.start(not args.mainnet) + print() + # Run the stages results = {} for s in args.stage: - results.update(run_stage(s)) + results.update(run_stage(s, zcash)) + + # Stop zcashd + print('Stopping zcashd...') + zcash.stop() passed = True print() From 01757cff2372030417beaf6794a8701d44f0ee36 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 21 Mar 2019 23:12:16 +1300 Subject: [PATCH 033/725] qa: Simple smoke tests --- qa/zcash/smoke_tests.py | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index 0f58f1e25..c385bf412 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -107,14 +107,64 @@ SMOKE_TESTS = [ ] +# +# Test helpers +# + +def run_cmd(results, case, zcash, name, args=[]): + print('-----') + print('%s $ zcash-cli %s %s' % ( + case.ljust(3), + name, + ' '.join([str(arg) for arg in args], + ))) + try: + res = zcash.__getattr__(name)(*args) + print(res) + print() + if results is not None and len(case) > 0: + results[case] = True + return res + except JSONRPCException as e: + print('error code: %d' % e.error['code']) + print('error message:') + print(e.error['message']) + if results is not None and len(case) > 0: + results[case] = False + return None + + +# +# Test runners +# + +def simple_commands(zcash): + results = {} + run_cmd(results, '2a', zcash, 'getinfo'), + run_cmd(results, '2b', zcash, 'help'), + run_cmd(results, '5o', zcash, 'getwalletinfo'), + run_cmd(results, '6a', zcash, 'getpeerinfo'), + run_cmd(results, '6b', zcash, 'getnetworkinfo'), + run_cmd(results, '6c', zcash, 'getdeprecationinfo'), + run_cmd(results, '6d', zcash, 'getconnectioncount'), + run_cmd(results, '6e', zcash, 'getaddednodeinfo', [False]), + run_cmd(results, '7a', zcash, 'getblocksubsidy'), + run_cmd(results, '7c', zcash, 'getmininginfo'), + run_cmd(results, '7d', zcash, 'getnetworkhashps'), + run_cmd(results, '7e', zcash, 'getnetworksolps'), + return results + + # # Test stages # STAGES = [ + 'simple-commands', ] STAGE_COMMANDS = { + 'simple-commands': simple_commands, } def run_stage(stage, zcash): From eeecb3bf8c2dd386a9704240842303dac4ff25eb Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Fri, 22 Mar 2019 02:25:55 +1300 Subject: [PATCH 034/725] qa: Transaction chain smoke test --- qa/zcash/smoke_tests.py | 437 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 437 insertions(+) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index c385bf412..1cb5928a4 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -16,10 +16,14 @@ sys.path.insert( ) import argparse +import decimal import subprocess +import time from authproxy import AuthServiceProxy, JSONRPCException +DEFAULT_FEE = decimal.Decimal('0.0001') + # # Smoke test definitions # @@ -133,6 +137,104 @@ def run_cmd(results, case, zcash, name, args=[]): results[case] = False return None +def wait_for_balance(zcash, zaddr, expected=None, timeout=900): + print('Waiting for funds to %s...' % zaddr) + if expected is not None: + print('Expecting %s' % expected) + balance = zcash.z_getbalance(zaddr) + for _ in iter(range(timeout)): + balance = zcash.z_getbalance(zaddr) + if expected is None and balance != 0: + break + if expected is not None and balance == expected: + break + time.sleep(1) + print('Received %s' % balance) + return balance + +def wait_and_check_balance(results, case, zcash, addr, expected): + balance = wait_for_balance(zcash, addr, expected) + if balance != expected and results is not None and len(case) > 0: + results[case] = False + return balance + +def wait_for_txid_operation(zcash, opid, timeout=300): + print('Waiting for async operation %s' % opid) + result = None + for _ in iter(range(timeout)): + results = zcash.z_getoperationresult([opid]) + if len(results) > 0: + result = results[0] + break + time.sleep(1) + + status = result['status'] + if status == 'failed': + print('Operation failed') + print(result['error']['message']) + return None + elif status == 'success': + txid = result['result']['txid'] + print(txid) + return txid + +def async_txid_cmd(results, case, zcash, name, args=[]): + opid = run_cmd(results, case, zcash, name, args) + # Some async commands return a dictionary containing the opid + if type(opid) == type({}): + opid = opid['opid'] + if opid is None: + if results is not None and len(case) > 0: + results[case] = False + return None + + txid = wait_for_txid_operation(zcash, opid) + if txid is None: + if results is not None and len(case) > 0: + results[case] = False + return txid + +def z_sendmany(results, case, zcash, from_addr, recipients): + return async_txid_cmd(results, case, zcash, 'z_sendmany', [ + from_addr, + [{ + 'address': to_addr, + 'amount': amount, + } for (to_addr, amount) in recipients] + ]) + +def check_z_sendmany(results, case, zcash, from_addr, recipients): + txid = z_sendmany(results, case, zcash, from_addr, recipients) + if txid is None: + return decimal.Decimal('0') + return [wait_and_check_balance(results, case, zcash, to_addr, amount) for (to_addr, amount) in recipients] + +def check_z_sendmany_parallel(results, zcash, runs): + # First attempt to create all the transactions + txids = [(run, z_sendmany(results, run[0], zcash, run[1], run[2])) for run in runs] + # Then wait for balance updates caused by successful transactions + return [ + wait_and_check_balance(results, run[0], zcash, to_addr, amount) if txid is not None else decimal.Decimal('0') + for (run, txid) in txids + for (to_addr, amount) in run[2]] + +def z_mergetoaddress(results, case, zcash, from_addrs, to_addr): + return async_txid_cmd(results, case, zcash, 'z_mergetoaddress', [from_addrs, to_addr]) + +def check_z_mergetoaddress(results, case, zcash, from_addrs, to_addr, amount): + txid = z_mergetoaddress(results, case, zcash, from_addrs, to_addr) + if txid is None: + return decimal.Decimal('0') + return wait_and_check_balance(results, case, zcash, to_addr, amount) + +def check_z_mergetoaddress_parallel(results, zcash, runs): + # First attempt to create all the transactions + txids = [(run, z_mergetoaddress(results, run[0], zcash, run[1], run[2])) for run in runs] + # Then wait for balance updates caused by successful transactions + return [ + wait_and_check_balance(results, run[0], zcash, run[2], run[3]) if txid is not None else decimal.Decimal('0') + for (run, txid) in txids] + # # Test runners @@ -154,6 +256,339 @@ def simple_commands(zcash): run_cmd(results, '7e', zcash, 'getnetworksolps'), return results +def transaction_chain(zcash): + results = {} + + # Generate the various addresses we will use + sprout_zaddr_1 = run_cmd(results, '3a', zcash, 'z_getnewaddress', ['sprout']) + sprout_zaddr_2 = run_cmd(results, '3b', zcash, 'z_getnewaddress', ['sprout']) + sprout_zaddr_3 = run_cmd(results, '3b', zcash, 'z_getnewaddress', ['sprout']) + taddr_1 = run_cmd(results, '3c', zcash, 'getnewaddress') + taddr_2 = run_cmd(results, '3d', zcash, 'getnewaddress') + taddr_3 = run_cmd(results, '3d', zcash, 'getnewaddress') + taddr_4 = run_cmd(results, '3d', zcash, 'getnewaddress') + taddr_5 = run_cmd(results, '3d', zcash, 'getnewaddress') + sapling_zaddr_1 = run_cmd(results, '3e', zcash, 'z_getnewaddress', ['sapling']) + sapling_zaddr_2 = run_cmd(results, '3f', zcash, 'z_getnewaddress', ['sapling']) + sapling_zaddr_3 = run_cmd(results, '3f', zcash, 'z_getnewaddress', ['sapling']) + + # Check that the zaddrs are all listed + zaddrs = run_cmd(results, '5a', zcash, 'z_listaddresses') + if (sprout_zaddr_1 not in zaddrs or + sprout_zaddr_2 not in zaddrs or + sapling_zaddr_1 not in zaddrs or + sapling_zaddr_2 not in zaddrs): + results['5a'] = False + + # Validate the addresses + ret = run_cmd(results, '5b', zcash, 'z_validateaddress', [sprout_zaddr_1]) + if not ret['isvalid'] or ret['type'] != 'sprout': + results['5b'] = False + + ret = run_cmd(results, '5c', zcash, 'z_validateaddress', [sapling_zaddr_1]) + if not ret['isvalid'] or ret['type'] != 'sapling': + results['5c'] = False + + # Set up beginning and end of the chain + print('#') + print('# Initialising transaction chain') + print('#') + print() + chain_end = input('Type or paste transparent address where leftover funds should be sent: ') + if not zcash.validateaddress(chain_end)['isvalid']: + print('Invalid transparent address') + return results + print() + print('Please send at least 0.01 ZEC/TAZ to the following address:') + print(sprout_zaddr_1) + print() + input('Press any key once the funds have been sent.') + print() + + # Wait to receive starting balance + sprout_balance = wait_for_balance(zcash, sprout_zaddr_1) + starting_balance = sprout_balance + + # + # Start the transaction chain! + # + print() + print('#') + print('# Starting transaction chain') + print('#') + print() + try: + # + # First, split the funds across all three pools + # + + # Sprout -> taddr + taddr_balance = check_z_sendmany( + results, '4c', zcash, sprout_zaddr_1, [(taddr_1, (starting_balance / 10) * 6)])[0] + sprout_balance -= taddr_balance + DEFAULT_FEE + + balance = run_cmd(results, '5f', zcash, 'z_getbalance', [sprout_zaddr_1]) + if balance != sprout_balance: + results['5f'] = False + + # taddr -> Sapling + # Send it all here because z_sendmany pick a new t-addr for change + sapling_balance = check_z_sendmany( + results, '4f', zcash, taddr_1, [(sapling_zaddr_1, taddr_balance - DEFAULT_FEE)])[0] + taddr_balance = 0 + + # Sapling -> taddr + taddr_balance = check_z_sendmany( + results, '4i', zcash, sapling_zaddr_1, [(taddr_1, (starting_balance / 10) * 3)])[0] + sapling_balance -= taddr_balance + DEFAULT_FEE + + # + # Intra-pool tests + # + + # Sprout -> same Sprout + # Sapling -> same Sapling + (sprout_balance, sapling_balance) = check_z_sendmany_parallel(results, zcash, [ + ('4a', sprout_zaddr_1, [(sprout_zaddr_1, sprout_balance - DEFAULT_FEE)]), + ('4g', sapling_zaddr_1, [(sapling_zaddr_1, sapling_balance - DEFAULT_FEE)]), + ]) + + # Sprout -> different Sprout + # taddr -> different taddr + # Sapling -> different Sapling + (sprout_balance, taddr_balance, sapling_balance) = check_z_sendmany_parallel(results, zcash, [ + ('4b', sprout_zaddr_1, [(sprout_zaddr_2, sprout_balance - DEFAULT_FEE)]), + ('4e', taddr_1, [(taddr_2, taddr_balance - DEFAULT_FEE)]), + ('4h', sapling_zaddr_1, [(sapling_zaddr_2, sapling_balance - DEFAULT_FEE)]), + ]) + + # Sprout -> multiple Sprout + # taddr -> multiple taddr + # Sapling -> multiple Sapling + check_z_sendmany_parallel(results, zcash, [ + ('4k', sprout_zaddr_2, [ + (sprout_zaddr_1, starting_balance / 10), + (sprout_zaddr_3, starting_balance / 10), + ]), + ('4p', taddr_2, [ + (taddr_1, starting_balance / 10), + (taddr_3, taddr_balance - (starting_balance / 10) - DEFAULT_FEE), + ]), + ('4t', sapling_zaddr_2, [ + (sapling_zaddr_1, starting_balance / 10), + (sapling_zaddr_3, starting_balance / 10), + ]), + ]) + sprout_balance -= DEFAULT_FEE + taddr_balance -= DEFAULT_FEE + sapling_balance -= DEFAULT_FEE + + # multiple Sprout -> Sprout + # multiple Sapling -> Sapling + # multiple taddr -> taddr + check_z_mergetoaddress_parallel(results, zcash, [ + ('4ff', [sprout_zaddr_1, sprout_zaddr_3], sprout_zaddr_2, sprout_balance - DEFAULT_FEE), + ('4gg', [sapling_zaddr_1, sapling_zaddr_3], sapling_zaddr_2, sapling_balance - DEFAULT_FEE), + ('', [taddr_1, taddr_3], taddr_2, taddr_balance - DEFAULT_FEE), + ]) + sprout_balance -= DEFAULT_FEE + sapling_balance -= DEFAULT_FEE + taddr_balance -= DEFAULT_FEE + + # + # Now test a bunch of failing cases + # + + # Sprout -> Sapling + txid = z_sendmany(results, '4j', zcash, sprout_zaddr_2, [(sapling_zaddr_1, sprout_balance - DEFAULT_FEE)]) + if txid is not None: + print('Should have failed') + return results + + # Sprout -> taddr and Sapling + txid = z_sendmany(results, '4n', zcash, sprout_zaddr_2, [ + (taddr_2, starting_balance / 10), + (sapling_zaddr_1, starting_balance / 10), + ]) + if txid is not None: + print('Should have failed') + return results + + # Sprout -> multiple Sapling + txid = z_sendmany(results, '4o', zcash, sprout_zaddr_2, [ + (sapling_zaddr_1, starting_balance / 10), + (sapling_zaddr_2, starting_balance / 10), + ]) + if txid is not None: + print('Should have failed') + return results + + # taddr -> Sprout and Sapling + txid = z_sendmany(results, '4s', zcash, taddr_2, [ + (sprout_zaddr_1, starting_balance / 10), + (sapling_zaddr_1, starting_balance / 10), + ]) + if txid is not None: + print('Should have failed') + return results + + # Sapling -> multiple Sprout + txid = z_sendmany(results, '4u', zcash, sapling_zaddr_2, [ + (sprout_zaddr_1, starting_balance / 10), + (sprout_zaddr_2, starting_balance / 10), + ]) + if txid is not None: + print('Should have failed') + return results + + # Sapling -> Sapling and Sprout + txid = z_sendmany(results, '4x', zcash, sapling_zaddr_2, [ + (sapling_zaddr_1, starting_balance / 10), + (sprout_zaddr_1, starting_balance / 10), + ]) + if txid is not None: + print('Should have failed') + return results + + # multiple same taddr -> Sprout + txid = z_mergetoaddress(results, '4aa', zcash, [taddr_2, taddr_2], sprout_zaddr_2) + if txid is not None: + print('Should have failed') + return results + + # + # Inter-pool tests + # + + # Sprout -> taddr and Sprout + # Sapling -> taddr and Sapling + check_z_sendmany_parallel(results, zcash, [ + ('4m', sprout_zaddr_2, [ + (taddr_1, starting_balance / 10), + (sprout_zaddr_1, starting_balance / 10), + ]), + ('4w', sapling_zaddr_2, [ + (taddr_3, starting_balance / 10), + (sapling_zaddr_1, starting_balance / 10), + ]), + ]) + sprout_balance -= (starting_balance / 10) + DEFAULT_FEE + sapling_balance -= (starting_balance / 10) + DEFAULT_FEE + taddr_balance += (starting_balance / 10) * 2 + + # taddr and Sprout -> Sprout + # taddr and Sapling -> Sapling + check_z_mergetoaddress_parallel(results, zcash, [ + ('4dd', [taddr_1, sprout_zaddr_1], sprout_zaddr_2, sprout_balance + (starting_balance / 10) - DEFAULT_FEE), + ('4ee', [taddr_3, sapling_zaddr_1], sapling_zaddr_2, sapling_balance + (starting_balance / 10) - DEFAULT_FEE), + ]) + sprout_balance += (starting_balance / 10) - DEFAULT_FEE + sapling_balance += (starting_balance / 10) - DEFAULT_FEE + taddr_balance -= (starting_balance / 10) * 2 + + # Sprout -> multiple taddr + # Sapling -> multiple taddr + check_z_sendmany_parallel(results, zcash, [ + ('4l', sprout_zaddr_2, [ + (taddr_1, (starting_balance / 10)), + (taddr_3, (starting_balance / 10)), + ]), + ('4v', sapling_zaddr_2, [ + (taddr_4, (starting_balance / 10)), + (taddr_5, (starting_balance / 10)), + ]), + ]) + sprout_balance -= ((starting_balance / 10) * 2) + DEFAULT_FEE + sapling_balance -= ((starting_balance / 10) * 2) + DEFAULT_FEE + taddr_balance += (starting_balance / 10) * 4 + + # multiple taddr -> Sprout + # multiple taddr -> Sapling + check_z_mergetoaddress_parallel(results, zcash, [ + ('4z', [taddr_1, taddr_3], sprout_zaddr_2, sprout_balance + ((starting_balance / 10) * 2) - DEFAULT_FEE), + ('4bb', [taddr_4, taddr_5], sapling_zaddr_2, sapling_balance + ((starting_balance / 10) * 2) - DEFAULT_FEE), + ]) + sprout_balance += ((starting_balance / 10) * 2) - DEFAULT_FEE + sapling_balance += ((starting_balance / 10) * 2) - DEFAULT_FEE + taddr_balance -= (starting_balance / 10) * 4 + + # taddr -> Sprout + check_z_sendmany_parallel(results, zcash, [ + ('4d', taddr_2, [(sprout_zaddr_3, taddr_balance - DEFAULT_FEE)]), + ]) + sprout_balance += taddr_balance - DEFAULT_FEE + taddr_balance = 0 + + # multiple Sprout -> taddr + # multiple Sapling -> taddr + check_z_mergetoaddress_parallel(None, zcash, [ + ('', [sprout_zaddr_1, sprout_zaddr_2, sprout_zaddr_3], taddr_1, sprout_balance - DEFAULT_FEE), + ('', [sapling_zaddr_1, sapling_zaddr_2, sapling_zaddr_3], taddr_2, sapling_balance - DEFAULT_FEE), + ]) + taddr_balance = sprout_balance + sapling_balance - (2 * DEFAULT_FEE) + sprout_balance = 0 + sapling_balance = 0 + + # taddr -> multiple Sprout + # taddr -> multiple Sapling + taddr_1_balance = zcash.z_getbalance(taddr_1) + taddr_2_balance = zcash.z_getbalance(taddr_2) + check_z_sendmany_parallel(results, zcash, [ + ('4q', taddr_1, [ + (sprout_zaddr_1, (starting_balance / 10)), + (sprout_zaddr_2, taddr_1_balance - (starting_balance / 10) - DEFAULT_FEE), + ]), + ('4r', taddr_2, [ + (sapling_zaddr_1, (starting_balance / 10)), + (sapling_zaddr_2, taddr_2_balance - (starting_balance / 10) - DEFAULT_FEE), + ]), + ]) + sprout_balance = taddr_1_balance - DEFAULT_FEE + sapling_balance = taddr_2_balance - DEFAULT_FEE + taddr_balance = 0 + + # multiple Sprout -> taddr + # multiple Sapling -> taddr + check_z_mergetoaddress_parallel(None, zcash, [ + ('', [sprout_zaddr_1, sprout_zaddr_2], taddr_1, sprout_balance - DEFAULT_FEE), + ('', [sapling_zaddr_1, sapling_zaddr_2], taddr_2, sapling_balance - DEFAULT_FEE), + ]) + taddr_balance = sprout_balance + sapling_balance - (2 * DEFAULT_FEE) + sprout_balance = 0 + sapling_balance = 0 + + # z_mergetoaddress taddr -> Sprout + # z_mergetoaddress taddr -> Sapling + taddr_1_balance = zcash.z_getbalance(taddr_1) + taddr_2_balance = zcash.z_getbalance(taddr_2) + check_z_mergetoaddress_parallel(results, zcash, [ + ('4y', [taddr_1], sprout_zaddr_1, taddr_1_balance - DEFAULT_FEE), + ('4cc', [taddr_2], sapling_zaddr_1, taddr_2_balance - DEFAULT_FEE), + ]) + sprout_balance = taddr_1_balance - DEFAULT_FEE + sapling_balance = taddr_2_balance - DEFAULT_FEE + taddr_balance = 0 + finally: + # + # End the chain by returning the remaining funds + # + print() + print('#') + print('# Finishing transaction chain') + print('#') + print() + print('Returning remaining balance minus fees') + for addr in [ + sprout_zaddr_1, sprout_zaddr_2, sprout_zaddr_3, + taddr_1, taddr_2, taddr_3, taddr_4, taddr_5, + sapling_zaddr_1, sapling_zaddr_2, sapling_zaddr_3, + ]: + balance = zcash.z_getbalance(addr) + if balance != 0: + z_sendmany(None, '', zcash, addr, [(chain_end, balance - DEFAULT_FEE)]) + + return results + # # Test stages @@ -161,10 +596,12 @@ def simple_commands(zcash): STAGES = [ 'simple-commands', + 'transaction-chain' ] STAGE_COMMANDS = { 'simple-commands': simple_commands, + 'transaction-chain': transaction_chain, } def run_stage(stage, zcash): From a6ff0a07943684474a05e3429577458bbe6b8227 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 12 Jun 2019 22:00:42 +0100 Subject: [PATCH 035/725] qa: Use slick-bitcoinrpc for smoke tests --- qa/zcash/smoke_tests.py | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index 1cb5928a4..eb3d54b87 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -3,24 +3,15 @@ # Execute the standard smoke tests for Zcash releases. # -# Add RPC test_framework to module search path: -import os -import sys -sys.path.insert( - 0, - os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - 'rpc-tests', - 'test_framework', - ) -) - import argparse import decimal +import os import subprocess +import sys import time -from authproxy import AuthServiceProxy, JSONRPCException +from slickrpc import Proxy +from slickrpc.exc import RpcException DEFAULT_FEE = decimal.Decimal('0.0001') @@ -129,10 +120,8 @@ def run_cmd(results, case, zcash, name, args=[]): if results is not None and len(case) > 0: results[case] = True return res - except JSONRPCException as e: - print('error code: %d' % e.error['code']) - print('error message:') - print(e.error['message']) + except RpcException as e: + print('ERROR: %s' % str(e)) if results is not None and len(case) > 0: results[case] = False return None @@ -691,9 +680,9 @@ class ZcashNode(object): url = 'http://%s@%s:%d' % (rpcuserpass, rpchost, rpcport) if timewait is not None: - self.__proxy = AuthServiceProxy(url, timeout=timewait) + self.__proxy = Proxy(url, timeout=timewait) else: - self.__proxy = AuthServiceProxy(url) + self.__proxy = Proxy(url) def stop(self): if self.__proxy is None: From 0261ccbbe8c071b6f1782256719f8fe413f6368f Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 12 Jun 2019 22:01:46 +0100 Subject: [PATCH 036/725] qa: Don't allow smoke tests with mainnet wallet.dat --- qa/zcash/smoke_tests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index eb3d54b87..2acdfb6ff 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -726,6 +726,11 @@ def main(): print("Invalid stage '%s' (choose from %s)" % (s, STAGES)) sys.exit(1) + # Don't allow using the default wallet.dat in mainnet mode + if args.mainnet and args.wallet == 'wallet.dat': + print('Cannot use wallet.dat as wallet file when running mainnet tests. Keep your funds safe!') + sys.exit(1) + # Start zcashd zcash = ZcashNode(args.datadir, args.wallet) print('Starting zcashd...') From a2ba414d69f9dffd2a53c45d78977aa34a1ff72e Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 12 Jun 2019 22:03:54 +0100 Subject: [PATCH 037/725] qa: Improve reliability of smoke tests --- qa/zcash/smoke_tests.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index 2acdfb6ff..de8358a12 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -130,16 +130,30 @@ def wait_for_balance(zcash, zaddr, expected=None, timeout=900): print('Waiting for funds to %s...' % zaddr) if expected is not None: print('Expecting %s' % expected) - balance = zcash.z_getbalance(zaddr) - for _ in iter(range(timeout)): + + unconfirmed_balance = zcash.z_getbalance(zaddr, 0) + print('Unconfirmed balance: %s' % unconfirmed_balance) + if expected is not None and unconfirmed_balance != expected: + print('WARNING: Unconfirmed balance does not match expected balance') + + ttl = timeout + while True: balance = zcash.z_getbalance(zaddr) - if expected is None and balance != 0: - break - if expected is not None and balance == expected: - break + if balance == unconfirmed_balance: + print('Received %s' % balance) + return balance + time.sleep(1) - print('Received %s' % balance) - return balance + ttl -= 1 + if ttl == 0: + # Ask user if they want to keep waiting + print() + ret = input('Do you wish to continue waiting? (Y/n) ') + if ret.to_lower() == 'n': + print('Address contained %s at timeout' % balance) + return balance + else: + ttl = 300 def wait_and_check_balance(results, case, zcash, addr, expected): balance = wait_for_balance(zcash, addr, expected) From a2db35137b932dcaec3a72bb884f63b248148616 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 12 Jun 2019 22:04:17 +0100 Subject: [PATCH 038/725] qa: Improve reliability of smoke test cleanup --- qa/zcash/smoke_tests.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index de8358a12..d2d99eca7 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -579,13 +579,19 @@ def transaction_chain(zcash): print('#') print('# Finishing transaction chain') print('#') - print() - print('Returning remaining balance minus fees') - for addr in [ + all_addrs = [ sprout_zaddr_1, sprout_zaddr_2, sprout_zaddr_3, taddr_1, taddr_2, taddr_3, taddr_4, taddr_5, sapling_zaddr_1, sapling_zaddr_2, sapling_zaddr_3, - ]: + ] + + print() + print('Waiting for all transactions to be mined') + [wait_for_balance(zcash, addr) for addr in all_addrs] + + print() + print('Returning remaining balance minus fees') + for addr in all_addrs: balance = zcash.z_getbalance(addr) if balance != 0: z_sendmany(None, '', zcash, addr, [(chain_end, balance - DEFAULT_FEE)]) From aa4b98a7c3ae2e804083391402e3298a86042207 Mon Sep 17 00:00:00 2001 From: Eirik Ogilvie-Wigley Date: Sat, 15 Jun 2019 15:03:58 -0600 Subject: [PATCH 039/725] Resolve decimal vs float issues --- qa/zcash/smoke_tests.py | 152 ++++++++++++++++++++-------------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index d2d99eca7..a0cc950ab 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -4,16 +4,16 @@ # import argparse -import decimal import os import subprocess import sys import time +from decimal import Decimal from slickrpc import Proxy from slickrpc.exc import RpcException -DEFAULT_FEE = decimal.Decimal('0.0001') +DEFAULT_FEE = Decimal('0.0001') # # Smoke test definitions @@ -128,18 +128,15 @@ def run_cmd(results, case, zcash, name, args=[]): def wait_for_balance(zcash, zaddr, expected=None, timeout=900): print('Waiting for funds to %s...' % zaddr) - if expected is not None: - print('Expecting %s' % expected) - - unconfirmed_balance = zcash.z_getbalance(zaddr, 0) - print('Unconfirmed balance: %s' % unconfirmed_balance) + unconfirmed_balance = Decimal(zcash.z_getbalance(zaddr, 0)).quantize(Decimal('1.00000000')) + print('Expecting: %s; Unconfirmed Balance: %s' % (expected, unconfirmed_balance)) if expected is not None and unconfirmed_balance != expected: print('WARNING: Unconfirmed balance does not match expected balance') ttl = timeout while True: - balance = zcash.z_getbalance(zaddr) - if balance == unconfirmed_balance: + balance = Decimal(zcash.z_getbalance(zaddr)).quantize(Decimal('1.00000000')) + if (expected is not None and balance == unconfirmed_balance) or (expected is None and balance > 0): print('Received %s' % balance) return balance @@ -209,7 +206,7 @@ def z_sendmany(results, case, zcash, from_addr, recipients): def check_z_sendmany(results, case, zcash, from_addr, recipients): txid = z_sendmany(results, case, zcash, from_addr, recipients) if txid is None: - return decimal.Decimal('0') + return [Decimal('0')] return [wait_and_check_balance(results, case, zcash, to_addr, amount) for (to_addr, amount) in recipients] def check_z_sendmany_parallel(results, zcash, runs): @@ -217,7 +214,7 @@ def check_z_sendmany_parallel(results, zcash, runs): txids = [(run, z_sendmany(results, run[0], zcash, run[1], run[2])) for run in runs] # Then wait for balance updates caused by successful transactions return [ - wait_and_check_balance(results, run[0], zcash, to_addr, amount) if txid is not None else decimal.Decimal('0') + wait_and_check_balance(results, run[0], zcash, to_addr, amount) if txid is not None else Decimal('0') for (run, txid) in txids for (to_addr, amount) in run[2]] @@ -227,7 +224,7 @@ def z_mergetoaddress(results, case, zcash, from_addrs, to_addr): def check_z_mergetoaddress(results, case, zcash, from_addrs, to_addr, amount): txid = z_mergetoaddress(results, case, zcash, from_addrs, to_addr) if txid is None: - return decimal.Decimal('0') + return Decimal('0') return wait_and_check_balance(results, case, zcash, to_addr, amount) def check_z_mergetoaddress_parallel(results, zcash, runs): @@ -235,7 +232,7 @@ def check_z_mergetoaddress_parallel(results, zcash, runs): txids = [(run, z_mergetoaddress(results, run[0], zcash, run[1], run[2])) for run in runs] # Then wait for balance updates caused by successful transactions return [ - wait_and_check_balance(results, run[0], zcash, run[2], run[3]) if txid is not None else decimal.Decimal('0') + wait_and_check_balance(results, run[0], zcash, run[2], run[3]) if txid is not None else Decimal('0') for (run, txid) in txids] @@ -327,10 +324,10 @@ def transaction_chain(zcash): # Sprout -> taddr taddr_balance = check_z_sendmany( - results, '4c', zcash, sprout_zaddr_1, [(taddr_1, (starting_balance / 10) * 6)])[0] + results, '4c', zcash, sprout_zaddr_1, [(taddr_1, (starting_balance / Decimal('10')) * Decimal('6'))])[0] sprout_balance -= taddr_balance + DEFAULT_FEE - balance = run_cmd(results, '5f', zcash, 'z_getbalance', [sprout_zaddr_1]) + balance = Decimal(run_cmd(results, '5f', zcash, 'z_getbalance', [sprout_zaddr_1])).quantize(Decimal('1.00000000')) if balance != sprout_balance: results['5f'] = False @@ -338,11 +335,11 @@ def transaction_chain(zcash): # Send it all here because z_sendmany pick a new t-addr for change sapling_balance = check_z_sendmany( results, '4f', zcash, taddr_1, [(sapling_zaddr_1, taddr_balance - DEFAULT_FEE)])[0] - taddr_balance = 0 + taddr_balance = Decimal('0') # Sapling -> taddr taddr_balance = check_z_sendmany( - results, '4i', zcash, sapling_zaddr_1, [(taddr_1, (starting_balance / 10) * 3)])[0] + results, '4i', zcash, sapling_zaddr_1, [(taddr_1, (starting_balance / Decimal('10')) * Decimal('3'))])[0] sapling_balance -= taddr_balance + DEFAULT_FEE # @@ -370,16 +367,16 @@ def transaction_chain(zcash): # Sapling -> multiple Sapling check_z_sendmany_parallel(results, zcash, [ ('4k', sprout_zaddr_2, [ - (sprout_zaddr_1, starting_balance / 10), - (sprout_zaddr_3, starting_balance / 10), + (sprout_zaddr_1, starting_balance / Decimal('10')), + (sprout_zaddr_3, starting_balance / Decimal('10')), ]), ('4p', taddr_2, [ - (taddr_1, starting_balance / 10), - (taddr_3, taddr_balance - (starting_balance / 10) - DEFAULT_FEE), + (taddr_1, starting_balance / Decimal('10')), + (taddr_3, taddr_balance - (starting_balance / Decimal('10')) - DEFAULT_FEE), ]), ('4t', sapling_zaddr_2, [ - (sapling_zaddr_1, starting_balance / 10), - (sapling_zaddr_3, starting_balance / 10), + (sapling_zaddr_1, starting_balance / Decimal('10')), + (sapling_zaddr_3, starting_balance / Decimal('10')), ]), ]) sprout_balance -= DEFAULT_FEE @@ -410,8 +407,8 @@ def transaction_chain(zcash): # Sprout -> taddr and Sapling txid = z_sendmany(results, '4n', zcash, sprout_zaddr_2, [ - (taddr_2, starting_balance / 10), - (sapling_zaddr_1, starting_balance / 10), + (taddr_2, starting_balance / Decimal('10')), + (sapling_zaddr_1, starting_balance / Decimal('10')), ]) if txid is not None: print('Should have failed') @@ -419,8 +416,8 @@ def transaction_chain(zcash): # Sprout -> multiple Sapling txid = z_sendmany(results, '4o', zcash, sprout_zaddr_2, [ - (sapling_zaddr_1, starting_balance / 10), - (sapling_zaddr_2, starting_balance / 10), + (sapling_zaddr_1, starting_balance / Decimal('10')), + (sapling_zaddr_2, starting_balance / Decimal('10')), ]) if txid is not None: print('Should have failed') @@ -428,8 +425,8 @@ def transaction_chain(zcash): # taddr -> Sprout and Sapling txid = z_sendmany(results, '4s', zcash, taddr_2, [ - (sprout_zaddr_1, starting_balance / 10), - (sapling_zaddr_1, starting_balance / 10), + (sprout_zaddr_1, starting_balance / Decimal('10')), + (sapling_zaddr_1, starting_balance / Decimal('10')), ]) if txid is not None: print('Should have failed') @@ -437,8 +434,8 @@ def transaction_chain(zcash): # Sapling -> multiple Sprout txid = z_sendmany(results, '4u', zcash, sapling_zaddr_2, [ - (sprout_zaddr_1, starting_balance / 10), - (sprout_zaddr_2, starting_balance / 10), + (sprout_zaddr_1, starting_balance / Decimal('10')), + (sprout_zaddr_2, starting_balance / Decimal('10')), ]) if txid is not None: print('Should have failed') @@ -446,8 +443,8 @@ def transaction_chain(zcash): # Sapling -> Sapling and Sprout txid = z_sendmany(results, '4x', zcash, sapling_zaddr_2, [ - (sapling_zaddr_1, starting_balance / 10), - (sprout_zaddr_1, starting_balance / 10), + (sapling_zaddr_1, starting_balance / Decimal('10')), + (sprout_zaddr_1, starting_balance / Decimal('10')), ]) if txid is not None: print('Should have failed') @@ -467,60 +464,60 @@ def transaction_chain(zcash): # Sapling -> taddr and Sapling check_z_sendmany_parallel(results, zcash, [ ('4m', sprout_zaddr_2, [ - (taddr_1, starting_balance / 10), - (sprout_zaddr_1, starting_balance / 10), + (taddr_1, starting_balance / Decimal('10')), + (sprout_zaddr_1, starting_balance / Decimal('10')), ]), ('4w', sapling_zaddr_2, [ - (taddr_3, starting_balance / 10), - (sapling_zaddr_1, starting_balance / 10), + (taddr_3, starting_balance / Decimal('10')), + (sapling_zaddr_1, starting_balance / Decimal('10')), ]), ]) - sprout_balance -= (starting_balance / 10) + DEFAULT_FEE - sapling_balance -= (starting_balance / 10) + DEFAULT_FEE - taddr_balance += (starting_balance / 10) * 2 + sprout_balance -= (starting_balance / Decimal('10')) + DEFAULT_FEE + sapling_balance -= (starting_balance / Decimal('10')) + DEFAULT_FEE + taddr_balance += (starting_balance / Decimal('10')) * Decimal('2') # taddr and Sprout -> Sprout # taddr and Sapling -> Sapling check_z_mergetoaddress_parallel(results, zcash, [ - ('4dd', [taddr_1, sprout_zaddr_1], sprout_zaddr_2, sprout_balance + (starting_balance / 10) - DEFAULT_FEE), - ('4ee', [taddr_3, sapling_zaddr_1], sapling_zaddr_2, sapling_balance + (starting_balance / 10) - DEFAULT_FEE), + ('4dd', [taddr_1, sprout_zaddr_1], sprout_zaddr_2, sprout_balance + (starting_balance / Decimal('10')) - DEFAULT_FEE), + ('4ee', [taddr_3, sapling_zaddr_1], sapling_zaddr_2, sapling_balance + (starting_balance / Decimal('10')) - DEFAULT_FEE), ]) - sprout_balance += (starting_balance / 10) - DEFAULT_FEE - sapling_balance += (starting_balance / 10) - DEFAULT_FEE - taddr_balance -= (starting_balance / 10) * 2 + sprout_balance += (starting_balance / Decimal('10')) - DEFAULT_FEE + sapling_balance += (starting_balance / Decimal('10')) - DEFAULT_FEE + taddr_balance -= (starting_balance / Decimal('10')) * Decimal('2') # Sprout -> multiple taddr # Sapling -> multiple taddr check_z_sendmany_parallel(results, zcash, [ ('4l', sprout_zaddr_2, [ - (taddr_1, (starting_balance / 10)), - (taddr_3, (starting_balance / 10)), + (taddr_1, (starting_balance / Decimal('10'))), + (taddr_3, (starting_balance / Decimal('10'))), ]), ('4v', sapling_zaddr_2, [ - (taddr_4, (starting_balance / 10)), - (taddr_5, (starting_balance / 10)), + (taddr_4, (starting_balance / Decimal('10'))), + (taddr_5, (starting_balance / Decimal('10'))), ]), ]) - sprout_balance -= ((starting_balance / 10) * 2) + DEFAULT_FEE - sapling_balance -= ((starting_balance / 10) * 2) + DEFAULT_FEE - taddr_balance += (starting_balance / 10) * 4 + sprout_balance -= ((starting_balance / Decimal('10')) * Decimal('2')) + DEFAULT_FEE + sapling_balance -= ((starting_balance / Decimal('10')) * Decimal('2')) + DEFAULT_FEE + taddr_balance += (starting_balance / Decimal('10')) * Decimal('4') # multiple taddr -> Sprout # multiple taddr -> Sapling check_z_mergetoaddress_parallel(results, zcash, [ - ('4z', [taddr_1, taddr_3], sprout_zaddr_2, sprout_balance + ((starting_balance / 10) * 2) - DEFAULT_FEE), - ('4bb', [taddr_4, taddr_5], sapling_zaddr_2, sapling_balance + ((starting_balance / 10) * 2) - DEFAULT_FEE), + ('4z', [taddr_1, taddr_3], sprout_zaddr_2, sprout_balance + ((starting_balance / Decimal('10')) * Decimal('2')) - DEFAULT_FEE), + ('4bb', [taddr_4, taddr_5], sapling_zaddr_2, sapling_balance + ((starting_balance / Decimal('10')) * Decimal('2')) - DEFAULT_FEE), ]) - sprout_balance += ((starting_balance / 10) * 2) - DEFAULT_FEE - sapling_balance += ((starting_balance / 10) * 2) - DEFAULT_FEE - taddr_balance -= (starting_balance / 10) * 4 + sprout_balance += ((starting_balance / Decimal('10')) * Decimal('2')) - DEFAULT_FEE + sapling_balance += ((starting_balance / Decimal('10')) * Decimal('2')) - DEFAULT_FEE + taddr_balance -= (starting_balance / Decimal('10')) * Decimal('4') # taddr -> Sprout check_z_sendmany_parallel(results, zcash, [ ('4d', taddr_2, [(sprout_zaddr_3, taddr_balance - DEFAULT_FEE)]), ]) sprout_balance += taddr_balance - DEFAULT_FEE - taddr_balance = 0 + taddr_balance = Decimal('0') # multiple Sprout -> taddr # multiple Sapling -> taddr @@ -529,26 +526,26 @@ def transaction_chain(zcash): ('', [sapling_zaddr_1, sapling_zaddr_2, sapling_zaddr_3], taddr_2, sapling_balance - DEFAULT_FEE), ]) taddr_balance = sprout_balance + sapling_balance - (2 * DEFAULT_FEE) - sprout_balance = 0 - sapling_balance = 0 + sprout_balance = Decimal('0') + sapling_balance = Decimal('0') # taddr -> multiple Sprout # taddr -> multiple Sapling - taddr_1_balance = zcash.z_getbalance(taddr_1) - taddr_2_balance = zcash.z_getbalance(taddr_2) + taddr_1_balance = Decimal(zcash.z_getbalance(taddr_1)).quantize(Decimal('1.00000000')) + taddr_2_balance = Decimal(zcash.z_getbalance(taddr_2)).quantize(Decimal('1.00000000')) check_z_sendmany_parallel(results, zcash, [ ('4q', taddr_1, [ - (sprout_zaddr_1, (starting_balance / 10)), - (sprout_zaddr_2, taddr_1_balance - (starting_balance / 10) - DEFAULT_FEE), + (sprout_zaddr_1, (starting_balance / Decimal('10'))), + (sprout_zaddr_2, taddr_1_balance - (starting_balance / Decimal('10')) - DEFAULT_FEE), ]), ('4r', taddr_2, [ - (sapling_zaddr_1, (starting_balance / 10)), - (sapling_zaddr_2, taddr_2_balance - (starting_balance / 10) - DEFAULT_FEE), + (sapling_zaddr_1, (starting_balance / Decimal('10'))), + (sapling_zaddr_2, taddr_2_balance - (starting_balance / Decimal('10')) - DEFAULT_FEE), ]), ]) sprout_balance = taddr_1_balance - DEFAULT_FEE sapling_balance = taddr_2_balance - DEFAULT_FEE - taddr_balance = 0 + taddr_balance = Decimal('0') # multiple Sprout -> taddr # multiple Sapling -> taddr @@ -556,21 +553,21 @@ def transaction_chain(zcash): ('', [sprout_zaddr_1, sprout_zaddr_2], taddr_1, sprout_balance - DEFAULT_FEE), ('', [sapling_zaddr_1, sapling_zaddr_2], taddr_2, sapling_balance - DEFAULT_FEE), ]) - taddr_balance = sprout_balance + sapling_balance - (2 * DEFAULT_FEE) - sprout_balance = 0 - sapling_balance = 0 + taddr_balance = sprout_balance + sapling_balance - (Decimal('2') * DEFAULT_FEE) + sprout_balance = Decimal('0') + sapling_balance = Decimal('0') # z_mergetoaddress taddr -> Sprout # z_mergetoaddress taddr -> Sapling - taddr_1_balance = zcash.z_getbalance(taddr_1) - taddr_2_balance = zcash.z_getbalance(taddr_2) + taddr_1_balance = Decimal(zcash.z_getbalance(taddr_1)).quantize(Decimal('1.00000000')) + taddr_2_balance = Decimal(zcash.z_getbalance(taddr_2)).quantize(Decimal('1.00000000')) check_z_mergetoaddress_parallel(results, zcash, [ ('4y', [taddr_1], sprout_zaddr_1, taddr_1_balance - DEFAULT_FEE), ('4cc', [taddr_2], sapling_zaddr_1, taddr_2_balance - DEFAULT_FEE), ]) sprout_balance = taddr_1_balance - DEFAULT_FEE sapling_balance = taddr_2_balance - DEFAULT_FEE - taddr_balance = 0 + taddr_balance = Decimal('0') finally: # # End the chain by returning the remaining funds @@ -587,13 +584,16 @@ def transaction_chain(zcash): print() print('Waiting for all transactions to be mined') - [wait_for_balance(zcash, addr) for addr in all_addrs] + for addr in all_addrs: + balance = Decimal(zcash.z_getbalance(addr, 0)).quantize(Decimal('1.00000000')) + if balance > 0: + wait_for_balance(zcash, addr, balance) print() print('Returning remaining balance minus fees') for addr in all_addrs: - balance = zcash.z_getbalance(addr) - if balance != 0: + balance = Decimal(zcash.z_getbalance(addr)).quantize(Decimal('1.00000000')) + if balance > 0: z_sendmany(None, '', zcash, addr, [(chain_end, balance - DEFAULT_FEE)]) return results From b21fcf37cb1f8e86813bfc6459ac753673246de3 Mon Sep 17 00:00:00 2001 From: Eirik Ogilvie-Wigley Date: Sat, 15 Jun 2019 20:09:52 -0600 Subject: [PATCH 040/725] Various improvements - Print time since start for each operation - Print traceback for any raised exceptions - Improve messages to user - Warning cleanups --- qa/zcash/smoke_tests.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index a0cc950ab..1579facc9 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -4,10 +4,12 @@ # import argparse +import datetime import os import subprocess import sys import time +import traceback from decimal import Decimal from slickrpc import Proxy @@ -101,13 +103,14 @@ SMOKE_TESTS = [ ('7e', True, True), # Run getnetworksolps ] +TIME_STARTED = datetime.datetime.now() # # Test helpers # def run_cmd(results, case, zcash, name, args=[]): - print('-----') + print('----- %s -----' % (datetime.datetime.now() - TIME_STARTED)) print('%s $ zcash-cli %s %s' % ( case.ljust(3), name, @@ -145,8 +148,9 @@ def wait_for_balance(zcash, zaddr, expected=None, timeout=900): if ttl == 0: # Ask user if they want to keep waiting print() + print('Balance: %s Expected: %s' % (balance, expected)) ret = input('Do you wish to continue waiting? (Y/n) ') - if ret.to_lower() == 'n': + if ret.lower() == 'n': print('Address contained %s at timeout' % balance) return balance else: @@ -175,13 +179,13 @@ def wait_for_txid_operation(zcash, opid, timeout=300): return None elif status == 'success': txid = result['result']['txid'] - print(txid) + print('txid: %s' % txid) return txid def async_txid_cmd(results, case, zcash, name, args=[]): opid = run_cmd(results, case, zcash, name, args) # Some async commands return a dictionary containing the opid - if type(opid) == type({}): + if isinstance(opid, dict): opid = opid['opid'] if opid is None: if results is not None and len(case) > 0: @@ -275,9 +279,9 @@ def transaction_chain(zcash): # Check that the zaddrs are all listed zaddrs = run_cmd(results, '5a', zcash, 'z_listaddresses') if (sprout_zaddr_1 not in zaddrs or - sprout_zaddr_2 not in zaddrs or - sapling_zaddr_1 not in zaddrs or - sapling_zaddr_2 not in zaddrs): + sprout_zaddr_2 not in zaddrs or + sapling_zaddr_1 not in zaddrs or + sapling_zaddr_2 not in zaddrs): results['5a'] = False # Validate the addresses @@ -302,7 +306,7 @@ def transaction_chain(zcash): print('Please send at least 0.01 ZEC/TAZ to the following address:') print(sprout_zaddr_1) print() - input('Press any key once the funds have been sent.') + input('Press Enter once the funds have been sent.') print() # Wait to receive starting balance @@ -568,6 +572,9 @@ def transaction_chain(zcash): sprout_balance = taddr_1_balance - DEFAULT_FEE sapling_balance = taddr_2_balance - DEFAULT_FEE taddr_balance = Decimal('0') + except Exception as e: + print('Error: %s' % e) + traceback.print_exc() finally: # # End the chain by returning the remaining funds @@ -730,8 +737,7 @@ def main(): parser.add_argument('--mainnet', action='store_true', help='Use mainnet instead of testnet') parser.add_argument('--wallet', default='wallet.dat', help='Wallet file to use (within data directory)') parser.add_argument('datadir', help='Data directory to use for smoke testing', default=None) - parser.add_argument('stage', nargs='*', default=STAGES, - help='One of %s'%STAGES) + parser.add_argument('stage', nargs='*', default=STAGES, help='One of %s' % STAGES) args = parser.parse_args() # Check for list @@ -753,6 +759,7 @@ def main(): # Start zcashd zcash = ZcashNode(args.datadir, args.wallet) + print('Start time: %s' % TIME_STARTED) print('Starting zcashd...') zcash.start(not args.mainnet) print() @@ -796,5 +803,6 @@ def main(): print("!!! One or more smoke test stages failed !!!") sys.exit(1) + if __name__ == '__main__': main() From a63c1d62790411d03df05a72d32fcc502b3f3a85 Mon Sep 17 00:00:00 2001 From: mdr0id Date: Wed, 22 Apr 2020 16:34:50 -0700 Subject: [PATCH 041/725] Add helpers for tapping and donating to the testnet faucet --- qa/zcash/smoke_tests.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index 1579facc9..484477843 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -6,6 +6,7 @@ import argparse import datetime import os +import requests import subprocess import sys import time @@ -16,6 +17,8 @@ from slickrpc import Proxy from slickrpc.exc import RpcException DEFAULT_FEE = Decimal('0.0001') +URL_FAUCET_DONATION = 'https://faucet.testnet.z.cash/donations' +URL_FAUCET_TAP = 'https://faucet.testnet.z.cash/' # # Smoke test definitions @@ -239,6 +242,41 @@ def check_z_mergetoaddress_parallel(results, zcash, runs): wait_and_check_balance(results, run[0], zcash, run[2], run[3]) if txid is not None else Decimal('0') for (run, txid) in txids] +def tap_zfaucet(addr): + with requests.Session() as session: + # Get token to request TAZ from faucet with a given zcash address + response = session.get(URL_FAUCET_TAP) + if response.status_code != 200: + print("Error establishing session at:", URL_FAUCET_TAP) + os.sys.exit(1) + csrftoken = response.cookies['csrftoken'] + + # Request TAZ from the faucet + data_params = dict(csrfmiddlewaretoken=csrftoken, address=addr) + response2 = session.post(URL_FAUCET_TAP, data=data_params, headers=dict(Referer=URL_FAUCET_TAP)) + if response2.status_code != 200: + print("Error tapping faucet at:", URL_FAUCET_TAP) + os.sys.exit(1) + +def get_zfaucet_addrs(): + with requests.Session() as session: + response = session.get(URL_FAUCET_DONATION) + if response.status_code != 200: + print("Error establishing session at:", URL_FAUCET_DONATION) + os.sys.exit(1) + data = response.json() + return data + +def get_zfaucet_taddr(): + return get_zfaucet_addrs()["t_address"] + +def get_zfaucet_zsapaddr(): + # At the time of writing this, it appears these(keys) are backwards + return get_zfaucet_addrs()["z_address_legacy"] + +def get_zfaucet_zsproutaddr(): + # At the time of writing this, it appears these(keys) are backwards + return get_zfaucet_addrs()["z_address_sapling"] # # Test runners From 3a1b1a2fa2d7b56fb90518c2f1506c6f85cba948 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 28 May 2020 20:45:25 +1200 Subject: [PATCH 042/725] metrics: Fix indents --- src/metrics.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/metrics.cpp b/src/metrics.cpp index df4a127b0..8c37d9f68 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -287,22 +287,22 @@ int printStats(bool mining) auto localsolps = GetLocalSolPS(); if (IsInitialBlockDownload(Params())) { - if (fReindex) { - int downloadPercent = nSizeReindexed * 100 / nFullSizeToReindex; - std::cout << " " << _("Reindexing blocks") << " | " << DisplaySize(nSizeReindexed) << " / " << DisplaySize(nFullSizeToReindex) << " (" << downloadPercent << "%, " << height << " " << _("blocks") << ")" << std::endl; - } else { - int nHeaders = currentHeadersHeight; - if (nHeaders < 0) - nHeaders = 0; - int netheight = currentHeadersHeight == -1 || currentHeadersTime == 0 ? - 0 : EstimateNetHeight(params, currentHeadersHeight, currentHeadersTime); - if (netheight < nHeaders) - netheight = nHeaders; - if (netheight <= 0) - netheight = 1; - int downloadPercent = height * 100 / netheight; - std::cout << " " << _("Downloading blocks") << " | " << height << " (" << nHeaders << " " << _("headers") << ") / ~" << netheight << " (" << downloadPercent << "%)" << std::endl; - } + if (fReindex) { + int downloadPercent = nSizeReindexed * 100 / nFullSizeToReindex; + std::cout << " " << _("Reindexing blocks") << " | " << DisplaySize(nSizeReindexed) << " / " << DisplaySize(nFullSizeToReindex) << " (" << downloadPercent << "%, " << height << " " << _("blocks") << ")" << std::endl; + } else { + int nHeaders = currentHeadersHeight; + if (nHeaders < 0) + nHeaders = 0; + int netheight = currentHeadersHeight == -1 || currentHeadersTime == 0 ? + 0 : EstimateNetHeight(params, currentHeadersHeight, currentHeadersTime); + if (netheight < nHeaders) + netheight = nHeaders; + if (netheight <= 0) + netheight = 1; + int downloadPercent = height * 100 / netheight; + std::cout << " " << _("Downloading blocks") << " | " << height << " (" << nHeaders << " " << _("headers") << ") / ~" << netheight << " (" << downloadPercent << "%)" << std::endl; + } } else { std::cout << " " << _("Block height") << " | " << height << std::endl; } From eb4ada98d2bbabbdecb066bea3639fa64c73a239 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 28 May 2020 21:10:19 +1200 Subject: [PATCH 043/725] metrics: Draw IBD progress bar showing headers and blocks --- src/metrics.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/metrics.cpp b/src/metrics.cpp index 8c37d9f68..b7e49b8bd 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -14,6 +14,7 @@ #include "utilmoneystr.h" #include "utilstrencodings.h" +#include #include #include #include @@ -302,6 +303,22 @@ int printStats(bool mining) netheight = 1; int downloadPercent = height * 100 / netheight; std::cout << " " << _("Downloading blocks") << " | " << height << " (" << nHeaders << " " << _("headers") << ") / ~" << netheight << " (" << downloadPercent << "%)" << std::endl; + + // Draw 50-character progress bar, which will fit into a 79-character line. + int blockChars = downloadPercent / 2; + int headerChars = (nHeaders * 50) / netheight; + std::cout << " | ["; + for (auto i : boost::irange(0, 50)) { + if (i < blockChars) { + std::cout << "█"; + } else if (i < headerChars) { + std::cout << "▄"; + } else { + std::cout << " "; + } + } + std::cout << "]" << std::endl; + lines++; } } else { std::cout << " " << _("Block height") << " | " << height << std::endl; From 96681695c957354e1a60fb623db443dccefc9f97 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 28 May 2020 21:11:49 +1200 Subject: [PATCH 044/725] metrics: Don't show "not mining" text for mainnet Mining with the zcashd built-in CPU miner is not useful work on mainnet at the current network difficulty. --- src/metrics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/metrics.cpp b/src/metrics.cpp index b7e49b8bd..30bd88322 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -373,7 +373,7 @@ int printMiningStatus(bool mining) } } lines++; - } else { + } else if (Params().NetworkIDString() != "main") { std::cout << _("You are currently not mining.") << std::endl; std::cout << _("To enable mining, add 'gen=1' to your zcash.conf and restart.") << std::endl; lines += 2; From c2144f7af735142c25870a760d32abccc1b4d1a1 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 28 May 2020 22:38:06 +1200 Subject: [PATCH 045/725] qa: Add --use-faucet flag to smoke tests --- qa/zcash/smoke_tests.py | 51 ++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index 484477843..cf7f8eb20 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -336,15 +336,21 @@ def transaction_chain(zcash): print('# Initialising transaction chain') print('#') print() - chain_end = input('Type or paste transparent address where leftover funds should be sent: ') - if not zcash.validateaddress(chain_end)['isvalid']: - print('Invalid transparent address') - return results - print() - print('Please send at least 0.01 ZEC/TAZ to the following address:') - print(sprout_zaddr_1) - print() - input('Press Enter once the funds have been sent.') + if zcash.use_faucet: + print('Tapping the testnet faucet for testing funds...') + chain_end = get_zfaucet_taddr() + tap_zfaucet(sprout_zaddr_1) + print('Done! Leftover funds will be sent back to the faucet.') + else: + chain_end = input('Type or paste transparent address where leftover funds should be sent: ') + if not zcash.validateaddress(chain_end)['isvalid']: + print('Invalid transparent address') + return results + print() + print('Please send at least 0.01 ZEC/TAZ to the following address:') + print(sprout_zaddr_1) + print() + input('Press Enter once the funds have been sent.') print() # Wait to receive starting balance @@ -683,20 +689,23 @@ def run_stage(stage, zcash): # class ZcashNode(object): - def __init__(self, datadir, wallet, zcashd=None, zcash_cli=None): + def __init__(self, args, zcashd=None, zcash_cli=None): if zcashd is None: zcashd = os.getenv('ZCASHD', 'zcashd') if zcash_cli is None: zcash_cli = os.getenv('ZCASHCLI', 'zcash-cli') - self.__datadir = datadir - self.__wallet = wallet + self.__datadir = args.datadir + self.__wallet = args.wallet + self.__testnet = not args.mainnet self.__zcashd = zcashd self.__zcash_cli = zcash_cli self.__process = None self.__proxy = None - def start(self, testnet=True, extra_args=None, timewait=None): + self.use_faucet = args.faucet + + def start(self, extra_args=None, timewait=None): if self.__proxy is not None: raise RuntimeError('Already started') @@ -713,7 +722,7 @@ class ZcashNode(object): '-experimentalfeatures', '-zmergetoaddress', ] - if testnet: + if self.__testnet: args.append('-testnet=1') if extra_args is not None: args.extend(extra_args) @@ -727,7 +736,7 @@ class ZcashNode(object): '-rpcpassword=%s' % rpcpassword, '-rpcwait', ] - if testnet: + if self.__testnet: cli_args.append('-testnet=1') cli_args.append('getblockcount') @@ -741,7 +750,7 @@ class ZcashNode(object): rpcuserpass = '%s:%s' % (rpcuser, rpcpassword) rpchost = '127.0.0.1' - rpcport = 18232 if testnet else 8232 + rpcport = 18232 if self.__testnet else 8232 url = 'http://%s@%s:%d' % (rpcuserpass, rpchost, rpcport) if timewait is not None: @@ -773,6 +782,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--list-stages', dest='list', action='store_true') parser.add_argument('--mainnet', action='store_true', help='Use mainnet instead of testnet') + parser.add_argument('--use-faucet', dest='faucet', action='store_true', help='Use testnet faucet as source of funds') parser.add_argument('--wallet', default='wallet.dat', help='Wallet file to use (within data directory)') parser.add_argument('datadir', help='Data directory to use for smoke testing', default=None) parser.add_argument('stage', nargs='*', default=STAGES, help='One of %s' % STAGES) @@ -795,11 +805,16 @@ def main(): print('Cannot use wallet.dat as wallet file when running mainnet tests. Keep your funds safe!') sys.exit(1) + # Testnet faucet cannot be used in mainnet mode + if args.mainnet and args.faucet: + print('Cannot use testnet faucet when running mainnet tests.') + sys.exit(1) + # Start zcashd - zcash = ZcashNode(args.datadir, args.wallet) + zcash = ZcashNode(args) print('Start time: %s' % TIME_STARTED) print('Starting zcashd...') - zcash.start(not args.mainnet) + zcash.start() print() # Run the stages From 4498eb26db1ce0ae4acac5d068b40e4d1b33ef37 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 28 May 2020 22:49:36 +1200 Subject: [PATCH 046/725] qa: Remove unused timeout configuration from wait_for_balance --- qa/zcash/smoke_tests.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index cf7f8eb20..7255a4caa 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -132,14 +132,15 @@ def run_cmd(results, case, zcash, name, args=[]): results[case] = False return None -def wait_for_balance(zcash, zaddr, expected=None, timeout=900): +def wait_for_balance(zcash, zaddr, expected=None): print('Waiting for funds to %s...' % zaddr) unconfirmed_balance = Decimal(zcash.z_getbalance(zaddr, 0)).quantize(Decimal('1.00000000')) print('Expecting: %s; Unconfirmed Balance: %s' % (expected, unconfirmed_balance)) if expected is not None and unconfirmed_balance != expected: print('WARNING: Unconfirmed balance does not match expected balance') - ttl = timeout + # Default timeout is 15 minutes + ttl = 900 while True: balance = Decimal(zcash.z_getbalance(zaddr)).quantize(Decimal('1.00000000')) if (expected is not None and balance == unconfirmed_balance) or (expected is None and balance > 0): @@ -157,6 +158,7 @@ def wait_for_balance(zcash, zaddr, expected=None, timeout=900): print('Address contained %s at timeout' % balance) return balance else: + # Wait another 5 minutes before asking again ttl = 300 def wait_and_check_balance(results, case, zcash, addr, expected): From 5a577c76307cd45fd4db207bbe3c563c13bf4d7d Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 28 May 2020 22:52:22 +1200 Subject: [PATCH 047/725] qa: Add --automate flag to smoke tests --- qa/zcash/smoke_tests.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/qa/zcash/smoke_tests.py b/qa/zcash/smoke_tests.py index 7255a4caa..0333a01b1 100755 --- a/qa/zcash/smoke_tests.py +++ b/qa/zcash/smoke_tests.py @@ -150,16 +150,20 @@ def wait_for_balance(zcash, zaddr, expected=None): time.sleep(1) ttl -= 1 if ttl == 0: - # Ask user if they want to keep waiting - print() - print('Balance: %s Expected: %s' % (balance, expected)) - ret = input('Do you wish to continue waiting? (Y/n) ') - if ret.lower() == 'n': - print('Address contained %s at timeout' % balance) - return balance - else: - # Wait another 5 minutes before asking again + if zcash.automated: + # Reset timeout ttl = 300 + else: + # Ask user if they want to keep waiting + print() + print('Balance: %s Expected: %s' % (balance, expected)) + ret = input('Do you wish to continue waiting? (Y/n) ') + if ret.lower() == 'n': + print('Address contained %s at timeout' % balance) + return balance + else: + # Wait another 5 minutes before asking again + ttl = 300 def wait_and_check_balance(results, case, zcash, addr, expected): balance = wait_for_balance(zcash, addr, expected) @@ -705,6 +709,7 @@ class ZcashNode(object): self.__process = None self.__proxy = None + self.automated = args.automate self.use_faucet = args.faucet def start(self, extra_args=None, timewait=None): @@ -782,6 +787,7 @@ class ZcashNode(object): def main(): parser = argparse.ArgumentParser() + parser.add_argument('--automate', action='store_true', help='Run the smoke tests without a user present') parser.add_argument('--list-stages', dest='list', action='store_true') parser.add_argument('--mainnet', action='store_true', help='Use mainnet instead of testnet') parser.add_argument('--use-faucet', dest='faucet', action='store_true', help='Use testnet faucet as source of funds') @@ -812,6 +818,15 @@ def main(): print('Cannot use testnet faucet when running mainnet tests.') sys.exit(1) + # Enforce correctly-configured automatic mode + if args.automate: + if args.mainnet: + print('Cannot yet automate mainnet tests.') + sys.exit(1) + if not args.faucet: + print('--automate requires --use-faucet') + sys.exit(1) + # Start zcashd zcash = ZcashNode(args) print('Start time: %s' % TIME_STARTED) From 540cdb34e2b20e8c86392f029111e353b444d2eb Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 28 Nov 2017 10:31:52 +0100 Subject: [PATCH 048/725] Add `-debuglogfile` option This patch adds an option to configure the name and/or directory of the debug log. The user can specify either a relative path, in which case the path is relative to the data directory. They can also specify an absolute path to put the log anywhere else in the file system. --- src/init.cpp | 8 ++++++-- src/util.cpp | 28 ++++++++++++++++++++++------ src/util.h | 4 +++- 3 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 8d6ecc545..4d5429ebb 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -355,6 +355,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-exportdir=", _("Specify directory to be used when exporting data")); strUsage += HelpMessageOpt("-dbcache=", strprintf(_("Set database cache size in megabytes (%d to %d, default: %d)"), nMinDbCache, nMaxDbCache, nDefaultDbCache)); strUsage += HelpMessageOpt("-loadblock=", _("Imports blocks from external blk000??.dat file on startup")); + strUsage += HelpMessageOpt("-debuglogfile=", strprintf(_("Specify location of debug log file: this can be an absolute path or a path relative to the data directory (default: %s)"), DEFAULT_DEBUGLOGFILE)); strUsage += HelpMessageOpt("-maxorphantx=", strprintf(_("Keep at most unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-par=", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); @@ -1127,8 +1128,11 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) // if (GetBoolArg("-shrinkdebugfile", !fDebug)) // ShrinkDebugFile(); - if (fPrintToDebugLog) - OpenDebugLog(); + if (fPrintToDebugLog) { + if (!OpenDebugLog()) { + return InitError(strprintf("Could not open debug log file %s", GetDebugLogPath().string())); + } + } LogPrintf("Using OpenSSL version %s\n", SSLeay_version(SSLEAY_VERSION)); #ifdef ENABLE_WALLET diff --git a/src/util.cpp b/src/util.cpp index 54a17123b..82d705ea8 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -102,6 +102,7 @@ using namespace std; const char * const BITCOIN_CONF_FILENAME = "zcash.conf"; const char * const BITCOIN_PID_FILENAME = "zcashd.pid"; +const char * const DEFAULT_DEBUGLOGFILE = "debug.log"; map mapArgs; map > mapMultiArgs; @@ -210,17 +211,31 @@ static void DebugPrintInit() vMsgsBeforeOpenLog = new list; } -void OpenDebugLog() +boost::filesystem::path GetDebugLogPath() +{ + boost::filesystem::path logfile(GetArg("-debuglogfile", DEFAULT_DEBUGLOGFILE)); + if (logfile.is_absolute()) { + return logfile; + } else { + return GetDataDir() / logfile; + } +} + +bool OpenDebugLog() { boost::call_once(&DebugPrintInit, debugPrintInitFlag); boost::mutex::scoped_lock scoped_lock(*mutexDebugLog); assert(fileout == NULL); assert(vMsgsBeforeOpenLog); - boost::filesystem::path pathDebug = GetDataDir() / "debug.log"; - fileout = fopen(pathDebug.string().c_str(), "a"); - if (fileout) setbuf(fileout, NULL); // unbuffered + boost::filesystem::path pathDebug = GetDebugLogPath(); + fileout = fopen(pathDebug.string().c_str(), "a"); + if (!fileout) { + return false; + } + + setbuf(fileout, nullptr); // unbuffered // dump buffered messages from before we opened the log while (!vMsgsBeforeOpenLog->empty()) { FileWriteStr(vMsgsBeforeOpenLog->front(), fileout); @@ -229,6 +244,7 @@ void OpenDebugLog() delete vMsgsBeforeOpenLog; vMsgsBeforeOpenLog = NULL; + return true; } bool LogAcceptCategory(const char* category) @@ -313,7 +329,7 @@ int LogPrintStr(const std::string &str) // reopen the log file, if requested if (fReopenDebugLog) { fReopenDebugLog = false; - boost::filesystem::path pathDebug = GetDataDir() / "debug.log"; + boost::filesystem::path pathDebug = GetDebugLogPath(); if (freopen(pathDebug.string().c_str(),"a",fileout) != NULL) setbuf(fileout, NULL); // unbuffered } @@ -771,7 +787,7 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) { void ShrinkDebugFile() { // Scroll debug.log if it's getting too big - boost::filesystem::path pathLog = GetDataDir() / "debug.log"; + boost::filesystem::path pathLog = GetDebugLogPath(); FILE* file = fopen(pathLog.string().c_str(), "r"); if (file && boost::filesystem::file_size(pathLog) > 10 * 1000000) { diff --git a/src/util.h b/src/util.h index 614a6bc01..d356cf8b4 100644 --- a/src/util.h +++ b/src/util.h @@ -32,6 +32,7 @@ static const bool DEFAULT_LOGTIMEMICROS = false; static const bool DEFAULT_LOGIPS = false; static const bool DEFAULT_LOGTIMESTAMPS = true; +extern const char * const DEFAULT_DEBUGLOGFILE; /** Signals for translation. */ class CTranslationInterface @@ -142,7 +143,8 @@ void ReadConfigFile(const std::string& confPath, std::map Date: Tue, 28 Nov 2017 15:38:46 +0100 Subject: [PATCH 049/725] test: Add test for `-debuglogfile` --- qa/pull-tester/rpc-tests.sh | 1 + qa/rpc-tests/feature_logging.py | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100755 qa/rpc-tests/feature_logging.py diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index 5ff4752c0..b6454a304 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -87,6 +87,7 @@ testScripts=( 'feature_zip221.py' 'upgrade_golden.py' 'post_heartwood_rollback.py' + 'feature_logging.py' ); testScriptsExt=( 'getblocktemplate_longpoll.py' diff --git a/qa/rpc-tests/feature_logging.py b/qa/rpc-tests/feature_logging.py new file mode 100755 index 000000000..24591e176 --- /dev/null +++ b/qa/rpc-tests/feature_logging.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Copyright (c) 2017 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test debug logging.""" + +import os + +from test_framework.test_framework import BitcoinTestFramework + +class LoggingTest(BitcoinTestFramework): + def set_test_params(self): + self.num_nodes = 1 + self.setup_clean_chain = True + + def run_test(self): + # test default log file name + assert os.path.isfile(os.path.join(self.nodes[0].datadir, "regtest", "debug.log")) + + # test alternative log file name in datadir + self.restart_node(0, ["-debuglogfile=foo.log"]) + assert os.path.isfile(os.path.join(self.nodes[0].datadir, "regtest", "foo.log")) + + # test alternative log file name outside datadir + tempname = os.path.join(self.options.tmpdir, "foo.log") + self.restart_node(0, ["-debuglogfile=%s" % tempname]) + assert os.path.isfile(tempname) + + # check that invalid log (relative) will cause error + self.stop_node(0) + self.assert_start_raises_init_error(0, ["-debuglogfile=ssdksjdf/sdasdfa/sdfsdfsfd"], + "Error: Could not open debug log file") + + # check that invalid log (absolute) will cause error + self.stop_node(0) + invalidname = os.path.join(self.options.tmpdir, "foo/foo.log") + self.assert_start_raises_init_error(0, ["-debuglogfile=%s" % invalidname], + "Error: Could not open debug log file") + + +if __name__ == '__main__': + LoggingTest().main() From 42b025705f3f525452fd7b0f0950c5e25a354662 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 28 Nov 2017 15:43:31 +0100 Subject: [PATCH 050/725] doc: Update release notes for `-debuglogfile` --- doc/release-notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/release-notes.md b/doc/release-notes.md index a29094b51..6a0fa40e6 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -4,3 +4,6 @@ release-notes at release time) Notable changes =============== +Changed command-line options +----------------------------- +- `-debuglogfile=` can be used to specify an alternative debug logging file. From 83087be3935265215c212cd096e4b92053c07d9d Mon Sep 17 00:00:00 2001 From: Anthony Towns Date: Fri, 1 Dec 2017 11:49:28 +1000 Subject: [PATCH 051/725] test: Add tests for `-debuglogfile` with subdirs --- qa/rpc-tests/feature_logging.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/feature_logging.py b/qa/rpc-tests/feature_logging.py index 24591e176..da4e7b039 100755 --- a/qa/rpc-tests/feature_logging.py +++ b/qa/rpc-tests/feature_logging.py @@ -27,15 +27,32 @@ class LoggingTest(BitcoinTestFramework): assert os.path.isfile(tempname) # check that invalid log (relative) will cause error + invdir = os.path.join(self.nodes[0].datadir, "regtest", "foo") + invalidname = os.path.join("foo", "foo.log") self.stop_node(0) - self.assert_start_raises_init_error(0, ["-debuglogfile=ssdksjdf/sdasdfa/sdfsdfsfd"], + self.assert_start_raises_init_error(0, ["-debuglogfile=%s" % (invalidname)], "Error: Could not open debug log file") + assert not os.path.isfile(os.path.join(invdir, "foo.log")) + + # check that invalid log (relative) works after path exists + self.stop_node(0) + os.mkdir(invdir) + self.start_node(0, ["-debuglogfile=%s" % (invalidname)]) + assert os.path.isfile(os.path.join(invdir, "foo.log")) # check that invalid log (absolute) will cause error self.stop_node(0) - invalidname = os.path.join(self.options.tmpdir, "foo/foo.log") + invdir = os.path.join(self.options.tmpdir, "foo") + invalidname = os.path.join(invdir, "foo.log") self.assert_start_raises_init_error(0, ["-debuglogfile=%s" % invalidname], "Error: Could not open debug log file") + assert not os.path.isfile(os.path.join(invdir, "foo.log")) + + # check that invalid log (absolute) works after path exists + self.stop_node(0) + os.mkdir(invdir) + self.start_node(0, ["-debuglogfile=%s" % (invalidname)]) + assert os.path.isfile(os.path.join(invdir, "foo.log")) if __name__ == '__main__': From 7c84f4e98a450414b6417ffafb4e2d844e9ebc9c Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 4 May 2020 15:09:39 -0300 Subject: [PATCH 052/725] fix test cases --- qa/rpc-tests/feature_logging.py | 34 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/qa/rpc-tests/feature_logging.py b/qa/rpc-tests/feature_logging.py index da4e7b039..cbb49ca96 100755 --- a/qa/rpc-tests/feature_logging.py +++ b/qa/rpc-tests/feature_logging.py @@ -6,6 +6,8 @@ import os +from test_framework.util import start_node, stop_node, assert_start_raises_init_error + from test_framework.test_framework import BitcoinTestFramework class LoggingTest(BitcoinTestFramework): @@ -15,45 +17,43 @@ class LoggingTest(BitcoinTestFramework): def run_test(self): # test default log file name - assert os.path.isfile(os.path.join(self.nodes[0].datadir, "regtest", "debug.log")) + assert os.path.isfile(os.path.join(self.options.tmpdir, "node0", "regtest", "debug.log")) # test alternative log file name in datadir - self.restart_node(0, ["-debuglogfile=foo.log"]) - assert os.path.isfile(os.path.join(self.nodes[0].datadir, "regtest", "foo.log")) + stop_node(self.nodes[0], 0) + self.nodes[0] = start_node(0, self.options.tmpdir, ["-debuglogfile=foo.log"]) + assert os.path.isfile(os.path.join(self.options.tmpdir, "node0", "regtest", "foo.log")) # test alternative log file name outside datadir tempname = os.path.join(self.options.tmpdir, "foo.log") - self.restart_node(0, ["-debuglogfile=%s" % tempname]) + stop_node(self.nodes[0], 0) + self.nodes[0] = start_node(0, self.options.tmpdir, ["-debuglogfile=%s" % tempname]) assert os.path.isfile(tempname) # check that invalid log (relative) will cause error - invdir = os.path.join(self.nodes[0].datadir, "regtest", "foo") + invdir = os.path.join(self.options.tmpdir, "node0", "regtest", "foo") invalidname = os.path.join("foo", "foo.log") - self.stop_node(0) - self.assert_start_raises_init_error(0, ["-debuglogfile=%s" % (invalidname)], - "Error: Could not open debug log file") + stop_node(self.nodes[0], 0) + assert_start_raises_init_error(0, "-debuglogfile=%s" % (invalidname), + "Error: Could not open debug log file") assert not os.path.isfile(os.path.join(invdir, "foo.log")) # check that invalid log (relative) works after path exists - self.stop_node(0) os.mkdir(invdir) - self.start_node(0, ["-debuglogfile=%s" % (invalidname)]) + self.nodes[0] = start_node(0, self.options.tmpdir, ["-debuglogfile=%s" % (invalidname)]) assert os.path.isfile(os.path.join(invdir, "foo.log")) # check that invalid log (absolute) will cause error - self.stop_node(0) + stop_node(self.nodes[0], 0) invdir = os.path.join(self.options.tmpdir, "foo") invalidname = os.path.join(invdir, "foo.log") - self.assert_start_raises_init_error(0, ["-debuglogfile=%s" % invalidname], - "Error: Could not open debug log file") - assert not os.path.isfile(os.path.join(invdir, "foo.log")) + assert_start_raises_init_error(0, "-debuglogfile=%s" % invalidname, + "Error: Could not open debug log file") # check that invalid log (absolute) works after path exists - self.stop_node(0) os.mkdir(invdir) - self.start_node(0, ["-debuglogfile=%s" % (invalidname)]) + self.nodes[0] = start_node(0, self.options.tmpdir, ["-debuglogfile=%s" % invalidname]) assert os.path.isfile(os.path.join(invdir, "foo.log")) - if __name__ == '__main__': LoggingTest().main() From 3b9bd2da2f0a1c0ecf0ea54ee1416b487904e51d Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 4 May 2020 15:11:29 -0300 Subject: [PATCH 053/725] fix sort of options --- src/init.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 4d5429ebb..1671dba90 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -352,10 +352,10 @@ std::string HelpMessage(HelpMessageMode mode) #endif } strUsage += HelpMessageOpt("-datadir=", _("Specify data directory")); - strUsage += HelpMessageOpt("-exportdir=", _("Specify directory to be used when exporting data")); strUsage += HelpMessageOpt("-dbcache=", strprintf(_("Set database cache size in megabytes (%d to %d, default: %d)"), nMinDbCache, nMaxDbCache, nDefaultDbCache)); - strUsage += HelpMessageOpt("-loadblock=", _("Imports blocks from external blk000??.dat file on startup")); strUsage += HelpMessageOpt("-debuglogfile=", strprintf(_("Specify location of debug log file: this can be an absolute path or a path relative to the data directory (default: %s)"), DEFAULT_DEBUGLOGFILE)); + strUsage += HelpMessageOpt("-exportdir=", _("Specify directory to be used when exporting data")); + strUsage += HelpMessageOpt("-loadblock=", _("Imports blocks from external blk000??.dat file on startup")); strUsage += HelpMessageOpt("-maxorphantx=", strprintf(_("Keep at most unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS)); strUsage += HelpMessageOpt("-par=", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"), -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS)); From 76cb4e8d0437be9f2843d7affac0f7573605dd12 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 29 May 2020 11:24:39 -0300 Subject: [PATCH 054/725] remove not needed comments from wallet.py --- qa/rpc-tests/wallet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index 626818a3b..4e2d157e1 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -265,11 +265,11 @@ class WalletTest (BitcoinTestFramework): sync_blocks(self.nodes) # tx should be added to balance because after restarting the nodes tx should be broadcast - assert_equal(self.nodes[2].getbalance(), Decimal('13.99800000')) #should not be - assert_equal(self.nodes[2].getbalance("*"), Decimal('13.99800000')) #should not be + assert_equal(self.nodes[2].getbalance(), Decimal('13.99800000')) + assert_equal(self.nodes[2].getbalance("*"), Decimal('13.99800000')) # check integer balances from getbalance - assert_equal(self.nodes[2].getbalance("*", 1, False, True), 1399800000) #should not be + assert_equal(self.nodes[2].getbalance("*", 1, False, True), 1399800000) # send from node 0 to node 2 taddr mytaddr = self.nodes[2].getnewaddress() From e7c829aa0e3cfe4336268e8576a0539d88cc4fc5 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Sat, 30 May 2020 10:34:03 -0300 Subject: [PATCH 055/725] update docs --- qa/rpc-tests/receivedby.py | 2 +- src/wallet/rpcwallet.cpp | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/qa/rpc-tests/receivedby.py b/qa/rpc-tests/receivedby.py index f60bdfa97..e40b4dd6f 100755 --- a/qa/rpc-tests/receivedby.py +++ b/qa/rpc-tests/receivedby.py @@ -84,7 +84,7 @@ class ReceivedByTest(BitcoinTestFramework): addr = self.nodes[1].getnewaddress() check_array_result(self.nodes[1].listreceivedbyaddress(0,True), {"address":addr}, - {"address":addr, "account":"", "amount":0, "confirmations":0, "amountZat":int("0"), "txids":[]}) + {"address":addr, "account":"", "amount":0, "confirmations":0, "amountZat":0, "txids":[]}) ''' getreceivedbyaddress Test diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 92a36bdb4..786202b36 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -604,9 +604,9 @@ UniValue getreceivedbyaddress(const UniValue& params, bool fHelp) "\nArguments:\n" "1. \"zcashaddress\" (string, required) The Zcash address for transactions.\n" "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" - "3. inZat (bool, optional, default=false) Get the result amount as an integer.\n" + "3. inZat (bool, optional, default=false) Get the result amount in " + MINOR_CURRENCY_UNIT + " (as an integer).\n" "\nResult:\n" - "amount (numeric) The total amount in " + CURRENCY_UNIT + " received at this address.\n" + "amount (numeric) The total amount in " + CURRENCY_UNIT + "(or " + MINOR_CURRENCY_UNIT + " if inZat is true) received at this address.\n" "\nExamples:\n" "\nThe amount from transactions with at least 1 confirmation\n" + HelpExampleCli("getreceivedbyaddress", "\"t14oHp2v54vfmdgQ3v3SNuQga8JKHTNi2a1\"") + @@ -670,9 +670,9 @@ UniValue getreceivedbyaccount(const UniValue& params, bool fHelp) "\nArguments:\n" "1. \"account\" (string, required) MUST be set to the empty string \"\" to represent the default account. Passing any other string will result in an error.\n" "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" - "3. inZat (bool, optional, default=false) Get the result amount as an integer.\n" + "3. inZat (bool, optional, default=false) Get the result amount in " + MINOR_CURRENCY_UNIT + " (as an integer).\n" "\nResult:\n" - "amount (numeric) The total amount in " + CURRENCY_UNIT + " received for this account.\n" + "amount (numeric) The total amount in " + CURRENCY_UNIT + "(or " + MINOR_CURRENCY_UNIT + " if inZat is true) received at this address.\n" "\nExamples:\n" "\nAmount received by the default account with at least 1 confirmation\n" + HelpExampleCli("getreceivedbyaccount", "\"\"") + @@ -766,9 +766,9 @@ UniValue getbalance(const UniValue& params, bool fHelp) "1. \"account\" (string, optional) DEPRECATED. If provided, it MUST be set to the empty string \"\" or to the string \"*\", either of which will give the total available balance. Passing any other string will result in an error.\n" "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" "3. includeWatchonly (bool, optional, default=false) Also include balance in watchonly addresses (see 'importaddress')\n" - "4. inZat (bool, optional, default=false) Get the result amount as an integer.\n" + "4. inZat (bool, optional, default=false) Get the result amount in " + MINOR_CURRENCY_UNIT + " (as an integer).\n" "\nResult:\n" - "amount (numeric) The total amount in " + CURRENCY_UNIT + " received for this account.\n" + "amount (numeric) The total amount in " + CURRENCY_UNIT + "(or " + MINOR_CURRENCY_UNIT + " if inZat is true) received at this address.\n" "\nExamples:\n" "\nThe total amount in the wallet\n" + HelpExampleCli("getbalance", "") + @@ -3496,9 +3496,9 @@ UniValue z_getbalance(const UniValue& params, bool fHelp) "\nArguments:\n" "1. \"address\" (string) The selected address. It may be a transparent or private address.\n" "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" - "3. inZat (bool, optional, default=false) Get the result amount as an integer.\n" + "3. inZat (bool, optional, default=false) Get the result amount in " + MINOR_CURRENCY_UNIT + " (as an integer).\n" "\nResult:\n" - "amount (numeric) The total amount in " + CURRENCY_UNIT + " received for this address.\n" + "amount (numeric) The total amount in " + CURRENCY_UNIT + "(or " + MINOR_CURRENCY_UNIT + " if inZat is true) received at this address.\n" "\nExamples:\n" "\nThe total amount received by address \"myaddress\"\n" + HelpExampleCli("z_getbalance", "\"myaddress\"") + From 93afebeef43597610ef8bad66c1ae1c9e69926b7 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Sat, 30 May 2020 12:18:43 -0300 Subject: [PATCH 056/725] add new parameters to rpc client and fix some bugs --- qa/rpc-tests/wallet.py | 2 +- src/rpc/client.cpp | 4 ++++ src/wallet/rpcwallet.cpp | 6 +++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index 4e2d157e1..fb3238460 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -282,7 +282,7 @@ class WalletTest (BitcoinTestFramework): assert_equal(mybalance, Decimal('10.0')) # check integer balances from z_getbalance - assert_equal(self.nodes[2].z_getbalance(mytaddr, 1, True), 10) + assert_equal(self.nodes[2].z_getbalance(mytaddr, 1, True), 1000000000) mytxdetails = self.nodes[2].gettransaction(mytxid) myvjoinsplits = mytxdetails["vjoinsplit"] diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index fe084b8f2..4887d0555 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -37,7 +37,9 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendtoaddress", 4 }, { "settxfee", 0 }, { "getreceivedbyaddress", 1 }, + { "getreceivedbyaddress", 2 }, { "getreceivedbyaccount", 1 }, + { "getreceivedbyaccount", 2 }, { "listreceivedbyaddress", 0 }, { "listreceivedbyaddress", 1 }, { "listreceivedbyaddress", 2 }, @@ -46,6 +48,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listreceivedbyaccount", 2 }, { "getbalance", 1 }, { "getbalance", 2 }, + { "getbalance", 3 }, { "getblockhash", 0 }, { "move", 2 }, { "move", 3 }, @@ -125,6 +128,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "z_listunspent", 2 }, { "z_listunspent", 3 }, { "z_getbalance", 1}, + { "z_getbalance", 2}, { "z_gettotalbalance", 0}, { "z_gettotalbalance", 1}, { "z_gettotalbalance", 2}, diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 786202b36..b954598b9 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -829,6 +829,10 @@ UniValue getbalance(const UniValue& params, bool fHelp) CAmount nBalance = GetAccountBalance(strAccount, nMinDepth, filter); + if (params.size() > 3 && params[3].get_bool()) { + return nBalance; + } + return ValueFromAmount(nBalance); } @@ -3541,7 +3545,7 @@ UniValue z_getbalance(const UniValue& params, bool fHelp) } // inZat - if (params.size() > 3 && params[3].get_bool()) { + if (params.size() > 2 && params[2].get_bool()) { return nBalance; } From 74039434a8e7f93a53e54042bc90b78b4f7f66d2 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 1 Jun 2020 09:47:02 -0300 Subject: [PATCH 057/725] initialize size_t --- src/txmempool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 9326a1089..1236288b8 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -813,7 +813,7 @@ size_t CTxMemPool::DynamicMemoryUsage() const { total += memusage::DynamicUsage(recentlyEvicted) + memusage::DynamicUsage(weightedTxTree); // Insight-related structures - size_t insight; + size_t insight = 0; insight += memusage::DynamicUsage(mapAddress); insight += memusage::DynamicUsage(mapAddressInserted); insight += memusage::DynamicUsage(mapSpent); From a79337c3ee6d93f83ac8b98db50666d41ab16b7f Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 2 Jun 2020 14:20:55 +1200 Subject: [PATCH 058/725] metrics: Switch to ANSI colour codes for progress bar We already assume that ANSI colour codes work for the metrics art, whereas the block characters have inconsistent support in fonts. --- src/metrics.cpp | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/metrics.cpp b/src/metrics.cpp index 30bd88322..dc7e721d4 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -307,17 +307,20 @@ int printStats(bool mining) // Draw 50-character progress bar, which will fit into a 79-character line. int blockChars = downloadPercent / 2; int headerChars = (nHeaders * 50) / netheight; - std::cout << " | ["; + // Start with background colour reversed for "full" bar. + std::cout << " | ["; for (auto i : boost::irange(0, 50)) { - if (i < blockChars) { - std::cout << "█"; - } else if (i < headerChars) { - std::cout << "▄"; - } else { - std::cout << " "; + if (i == headerChars) { + // Switch to normal background colour for "empty" bar. + std::cout << ""; + } else if (i == blockChars) { + // Switch to distinct colour for "headers" bar. + std::cout << ""; } + std::cout << " "; } - std::cout << "]" << std::endl; + // Ensure that colour is reset after the progress bar is printed. + std::cout << "]" << std::endl; lines++; } } else { From 12e169d09ff61fabe58ef0e053bb97098e42be4f Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 2 Jun 2020 14:23:03 +1200 Subject: [PATCH 059/725] metrics: Only print IBD progress bar on TTY Now that it is created from space characters, it is meaningless to print it to a log file. --- src/metrics.cpp | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/metrics.cpp b/src/metrics.cpp index dc7e721d4..e04a473c6 100644 --- a/src/metrics.cpp +++ b/src/metrics.cpp @@ -266,7 +266,7 @@ boost::optional SecondsLeftToNextEpoch(const Consensus::Params& params, } } -int printStats(bool mining) +int printStats(bool isScreen, bool mining) { // Number of lines that are always displayed int lines = 5; @@ -304,24 +304,26 @@ int printStats(bool mining) int downloadPercent = height * 100 / netheight; std::cout << " " << _("Downloading blocks") << " | " << height << " (" << nHeaders << " " << _("headers") << ") / ~" << netheight << " (" << downloadPercent << "%)" << std::endl; - // Draw 50-character progress bar, which will fit into a 79-character line. - int blockChars = downloadPercent / 2; - int headerChars = (nHeaders * 50) / netheight; - // Start with background colour reversed for "full" bar. - std::cout << " | ["; - for (auto i : boost::irange(0, 50)) { - if (i == headerChars) { - // Switch to normal background colour for "empty" bar. - std::cout << ""; - } else if (i == blockChars) { - // Switch to distinct colour for "headers" bar. - std::cout << ""; + if (isScreen) { + // Draw 50-character progress bar, which will fit into a 79-character line. + int blockChars = downloadPercent / 2; + int headerChars = (nHeaders * 50) / netheight; + // Start with background colour reversed for "full" bar. + std::cout << " | ["; + for (auto i : boost::irange(0, 50)) { + if (i == headerChars) { + // Switch to normal background colour for "empty" bar. + std::cout << ""; + } else if (i == blockChars) { + // Switch to distinct colour for "headers" bar. + std::cout << ""; + } + std::cout << " "; } - std::cout << " "; + // Ensure that colour is reset after the progress bar is printed. + std::cout << "]" << std::endl; + lines++; } - // Ensure that colour is reset after the progress bar is printed. - std::cout << "]" << std::endl; - lines++; } } else { std::cout << " " << _("Block height") << " | " << height << std::endl; @@ -604,7 +606,7 @@ void ThreadShowMetricsScreen() #endif if (loaded) { - lines += printStats(mining); + lines += printStats(isScreen, mining); lines += printMiningStatus(mining); } lines += printMetrics(cols, mining); From 916351be049a613adf2574a899edd402862f4494 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 2 Jun 2020 09:14:50 -0300 Subject: [PATCH 060/725] fix/improve docs Co-authored-by: str4d --- src/wallet/rpcwallet.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index b954598b9..75c273095 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -672,7 +672,7 @@ UniValue getreceivedbyaccount(const UniValue& params, bool fHelp) "2. minconf (numeric, optional, default=1) Only include transactions confirmed at least this many times.\n" "3. inZat (bool, optional, default=false) Get the result amount in " + MINOR_CURRENCY_UNIT + " (as an integer).\n" "\nResult:\n" - "amount (numeric) The total amount in " + CURRENCY_UNIT + "(or " + MINOR_CURRENCY_UNIT + " if inZat is true) received at this address.\n" + "amount (numeric) The total amount in " + CURRENCY_UNIT + "(or " + MINOR_CURRENCY_UNIT + " if inZat is true) received for this account.\n" "\nExamples:\n" "\nAmount received by the default account with at least 1 confirmation\n" + HelpExampleCli("getreceivedbyaccount", "\"\"") + @@ -768,7 +768,7 @@ UniValue getbalance(const UniValue& params, bool fHelp) "3. includeWatchonly (bool, optional, default=false) Also include balance in watchonly addresses (see 'importaddress')\n" "4. inZat (bool, optional, default=false) Get the result amount in " + MINOR_CURRENCY_UNIT + " (as an integer).\n" "\nResult:\n" - "amount (numeric) The total amount in " + CURRENCY_UNIT + "(or " + MINOR_CURRENCY_UNIT + " if inZat is true) received at this address.\n" + "amount (numeric) The total amount in " + CURRENCY_UNIT + "(or " + MINOR_CURRENCY_UNIT + " if inZat is true) received.\n" "\nExamples:\n" "\nThe total amount in the wallet\n" + HelpExampleCli("getbalance", "") + From 270a724c877f40aca61224e298931fbc7011d7fe Mon Sep 17 00:00:00 2001 From: Daira Hopwood Date: Wed, 3 Jun 2020 12:25:31 +0100 Subject: [PATCH 061/725] Remove unused import in qa/rpc-tests/listtransactions.py Co-authored-by: Dimitris Apostolou --- qa/rpc-tests/listtransactions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/qa/rpc-tests/listtransactions.py b/qa/rpc-tests/listtransactions.py index 990f9a605..6d14a3344 100755 --- a/qa/rpc-tests/listtransactions.py +++ b/qa/rpc-tests/listtransactions.py @@ -6,7 +6,6 @@ # Exercise the listtransactions API from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal from decimal import Decimal @@ -107,4 +106,3 @@ class ListTransactionsTest(BitcoinTestFramework): if __name__ == '__main__': ListTransactionsTest().main() - From 27b8a616de75c7e6be585493bcc6266598244e50 Mon Sep 17 00:00:00 2001 From: George Tankersley Date: Wed, 10 Jun 2020 21:10:43 -0400 Subject: [PATCH 062/725] Add ZF and gtank's DNS seeders --- src/chainparams.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 9ecb1eb52..cd53f0730 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -152,6 +152,8 @@ public: vSeeds.clear(); vSeeds.push_back(CDNSSeedData("z.cash", "dnsseed.z.cash")); // Zcash vSeeds.push_back(CDNSSeedData("str4d.xyz", "dnsseed.str4d.xyz")); // @str4d + vSeeds.push_back(CDNSSeedData("zfnd.org", "mainnet.seeder.zfnd.org")); // Zcash Foundation + vSeeds.push_back(CDNSSeedData("yolo.money", "mainnet.is.yolo.money")); // gtank // guarantees the first 2 characters, when base58 encoded, are "t1" base58Prefixes[PUBKEY_ADDRESS] = {0x1C,0xB8}; @@ -370,6 +372,8 @@ public: vFixedSeeds.clear(); vSeeds.clear(); vSeeds.push_back(CDNSSeedData("z.cash", "dnsseed.testnet.z.cash")); // Zcash + vSeeds.push_back(CDNSSeedData("zfnd.org", "testnet.seeder.zfnd.org")); // Zcash Foundation + vSeeds.push_back(CDNSSeedData("yolo.money", "testnet.is.yolo.money")); // gtank // guarantees the first 2 characters, when base58 encoded, are "tm" base58Prefixes[PUBKEY_ADDRESS] = {0x1D,0x25}; From 18a1448ac71a51531ead57a8bd83bf3fa493812d Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Fri, 12 Jun 2020 18:08:00 -0300 Subject: [PATCH 063/725] add timestamp to warnings --- src/init.cpp | 2 +- src/main.cpp | 8 ++++---- src/main.h | 4 ++-- src/rpc/mining.cpp | 6 +++++- src/rpc/misc.cpp | 6 +++++- src/rpc/net.cpp | 6 +++++- src/test/alert_tests.cpp | 40 ++++++++++++++++++++++++---------------- src/timedata.cpp | 2 +- src/warnings.cpp | 40 ++++++++++++++++++++++++---------------- src/warnings.h | 6 +++--- 10 files changed, 74 insertions(+), 46 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 1671dba90..02cc0f9fa 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -324,7 +324,7 @@ void OnRPCStopped() void OnRPCPreCommand(const CRPCCommand& cmd) { // Observe safe mode - string strWarning = GetWarnings("rpc"); + string strWarning = GetWarnings("rpc").second; if (strWarning != "" && !GetBoolArg("-disablesafemode", DEFAULT_DISABLE_SAFEMODE) && !cmd.okSafeMode) throw JSONRPCError(RPC_FORBIDDEN_BY_SAFE_MODE, string("Safe mode: ") + strWarning); diff --git a/src/main.cpp b/src/main.cpp index d11ca5bae..43dfc0b76 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -117,7 +117,7 @@ namespace { /** Abort with a message */ bool AbortNode(const std::string& strMessage, const std::string& userMessage="") { - SetMiscWarning(strMessage); + SetMiscWarning(strMessage, GetTime()); LogPrintf("*** %s\n", strMessage); uiInterface.ThreadSafeMessageBox( userMessage.empty() ? _("Error: A fatal internal error occurred, see debug.log for details") : userMessage, @@ -2585,18 +2585,18 @@ void PartitionCheck(bool (*initialDownloadCheck)(const CChainParams&), if (p <= alertThreshold && nBlocks < BLOCKS_EXPECTED) { // Many fewer blocks than expected: alert! - strWarning = strprintf(_("WARNING: check your network connection, %d blocks received in the last %d hours (%d expected)"), + strWarning = strprintf(_("WARNING: Check your network connection, %d blocks received in the last %d hours (%d expected)"), nBlocks, SPAN_HOURS, BLOCKS_EXPECTED); } else if (p <= alertThreshold && nBlocks > BLOCKS_EXPECTED) { // Many more blocks than expected: alert! - strWarning = strprintf(_("WARNING: abnormally high number of blocks generated, %d blocks received in the last %d hours (%d expected)"), + strWarning = strprintf(_("WARNING: Abnormally high number of blocks generated, %d blocks received in the last %d hours (%d expected)"), nBlocks, SPAN_HOURS, BLOCKS_EXPECTED); } if (!strWarning.empty()) { - SetMiscWarning(strWarning); + SetMiscWarning(strWarning, GetTime()); CAlert::Notify(strWarning, true); lastAlertTime = now; } diff --git a/src/main.h b/src/main.h index 193d06c56..12704a3a3 100644 --- a/src/main.h +++ b/src/main.h @@ -249,8 +249,8 @@ void ThreadScriptCheck(); void PartitionCheck(bool (*initialDownloadCheck)(const CChainParams&), CCriticalSection& cs, const CBlockIndex *const &bestHeader); /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload(const CChainParams& chainParams); -/** Format a string that describes several potential problems detected by the core */ -std::string GetWarnings(const std::string& strFor); +/** Pair of timestamp and formatted string that describes several potential problems detected by the core */ +std::pair GetWarnings(const std::string& strFor); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ bool GetTransaction(const uint256 &hash, CTransaction &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); /** Find the best known block, and make it the tip of the block chain */ diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 00de5f54d..b0e5d0476 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -341,7 +341,11 @@ UniValue getmininginfo(const UniValue& params, bool fHelp) obj.pushKV("currentblocksize", (uint64_t)nLastBlockSize); obj.pushKV("currentblocktx", (uint64_t)nLastBlockTx); obj.pushKV("difficulty", (double)GetNetworkDifficulty()); - obj.pushKV("errors", GetWarnings("statusbar")); + UniValue wobj(UniValue::VOBJ); + auto warnings = GetWarnings("statusbar"); + wobj.pushKV("timestamp", warnings.first); + wobj.pushKV("msg", warnings.second); + obj.pushKV("errors", wobj); obj.pushKV("genproclimit", (int)GetArg("-genproclimit", DEFAULT_GENERATE_THREADS)); obj.pushKV("localsolps" , getlocalsolps(params, false)); obj.pushKV("networksolps", getnetworksolps(params, false)); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index a6fb58f7e..6268936e0 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -105,7 +105,11 @@ UniValue getinfo(const UniValue& params, bool fHelp) obj.pushKV("paytxfee", ValueFromAmount(payTxFee.GetFeePerK())); #endif obj.pushKV("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); - obj.pushKV("errors", GetWarnings("statusbar")); + UniValue wobj(UniValue::VOBJ); + auto warnings = GetWarnings("statusbar"); + wobj.pushKV("timestamp", warnings.first); + wobj.pushKV("msg", warnings.second); + obj.pushKV("errors", wobj); return obj; } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 9f8b4cf72..a66eee589 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -488,7 +488,11 @@ UniValue getnetworkinfo(const UniValue& params, bool fHelp) } } obj.pushKV("localaddresses", localAddresses); - obj.pushKV("warnings", GetWarnings("statusbar")); + UniValue wobj(UniValue::VOBJ); + auto warnings = GetWarnings("statusbar"); + wobj.pushKV("timestamp", warnings.first); + wobj.pushKV("msg", warnings.second); + obj.pushKV("errors", wobj); return obj; } diff --git a/src/test/alert_tests.cpp b/src/test/alert_tests.cpp index c4ba04c37..da1c47885 100644 --- a/src/test/alert_tests.cpp +++ b/src/test/alert_tests.cpp @@ -395,17 +395,17 @@ BOOST_AUTO_TEST_CASE(AlertDisablesRPC) const std::vector& alertKey = Params(CBaseChainParams::MAIN).AlertKey(); // Command should work before alerts - BOOST_CHECK_EQUAL(GetWarnings("rpc"), ""); + BOOST_CHECK_EQUAL(GetWarnings("rpc").second, ""); // First alert should disable RPC alerts[7].ProcessAlert(alertKey, false); BOOST_CHECK_EQUAL(alerts[7].strRPCError, "RPC disabled"); - BOOST_CHECK_EQUAL(GetWarnings("rpc"), "RPC disabled"); + BOOST_CHECK_EQUAL(GetWarnings("rpc").second, "RPC disabled"); // Second alert should re-enable RPC alerts[8].ProcessAlert(alertKey, false); BOOST_CHECK_EQUAL(alerts[8].strRPCError, ""); - BOOST_CHECK_EQUAL(GetWarnings("rpc"), ""); + BOOST_CHECK_EQUAL(GetWarnings("rpc").second, ""); SetMockTime(0); mapAlerts.clear(); @@ -436,26 +436,30 @@ void PartitionAlertTestImpl(const Consensus::Params& params, int startTime, int // Test 1: chain with blocks every nPowTargetSpacing seconds, // as normal, no worries: - SetMiscWarning(""); + SetMiscWarning("", 0); PartitionCheck(falseFunc, csDummy, &indexDummy[799]); - BOOST_CHECK_EQUAL("", GetMiscWarning()); + BOOST_CHECK_EQUAL("", GetMiscWarning().second); // Test 2: go 3.5 hours without a block, expect a warning: now += 3*60*60+30*60; SetMockTime(now); - SetMiscWarning(""); + SetMiscWarning("", 0); PartitionCheck(falseFunc, csDummy, &indexDummy[799]); - std::string expectedSlowErr = strprintf("WARNING: check your network connection, %d blocks received in the last 4 hours (%d expected)", expectedSlow, expectedTotal); - BOOST_CHECK_EQUAL(expectedSlowErr, GetMiscWarning()); - BOOST_TEST_MESSAGE(std::string("Got alert text: ")+GetMiscWarning()); + std::string expectedSlowErr = strprintf("WARNING: Check your network connection, %d blocks received in the last 4 hours (%d expected)", expectedSlow, expectedTotal); + auto warning = GetMiscWarning(); + // advance 5 seconds so alert time will be in the past + SetMockTime(now + 5); + BOOST_CHECK_EQUAL(GetTime() - 5, warning.first); + BOOST_CHECK_EQUAL(expectedSlowErr, warning.second); + BOOST_TEST_MESSAGE(DateTimeStrFormat("%Y-%m-%d %H:%M:%S", warning.first) + " - Got alert text: " + warning.second); // Test 3: test the "partition alerts only go off once per day" // code: now += 60*10; SetMockTime(now); - SetMiscWarning(""); + SetMiscWarning("", 0); PartitionCheck(falseFunc, csDummy, &indexDummy[799]); - BOOST_CHECK_EQUAL("", GetMiscWarning()); + BOOST_CHECK_EQUAL("", GetMiscWarning().second); // Test 4: get 2.5 times as many blocks as expected: start = now + 60*60*24; // Pretend it is a day later @@ -466,12 +470,16 @@ void PartitionAlertTestImpl(const Consensus::Params& params, int startTime, int now = indexDummy[799].nTime + params.PoWTargetSpacing(0) * 2/5; SetMockTime(now); - SetMiscWarning(""); + SetMiscWarning("", 0); PartitionCheck(falseFunc, csDummy, &indexDummy[799]); - std::string expectedFastErr = strprintf("WARNING: abnormally high number of blocks generated, %d blocks received in the last 4 hours (%d expected)", expectedFast, expectedTotal); - BOOST_CHECK_EQUAL(expectedFastErr, GetMiscWarning()); - BOOST_TEST_MESSAGE(std::string("Got alert text: ")+GetMiscWarning()); - SetMiscWarning(""); + std::string expectedFastErr = strprintf("WARNING: Abnormally high number of blocks generated, %d blocks received in the last 4 hours (%d expected)", expectedFast, expectedTotal); + warning = GetMiscWarning(); + // advance 5 seconds so alert time will be in the past + SetMockTime(now + 5); + BOOST_CHECK_EQUAL(GetTime() - 5, warning.first); + BOOST_CHECK_EQUAL(expectedFastErr, warning.second); + BOOST_TEST_MESSAGE(DateTimeStrFormat("%Y-%m-%d %H:%M:%S", warning.first) + " - Got alert text: " + warning.second); + SetMiscWarning("", 0); SetMockTime(0); } diff --git a/src/timedata.cpp b/src/timedata.cpp index 2044105db..93e78839e 100644 --- a/src/timedata.cpp +++ b/src/timedata.cpp @@ -64,7 +64,7 @@ void CTimeWarning::Warn(size_t peersAhead, size_t peersBehind) } else { strMessage = _("Warning: Please check that your computer's date and time are correct! If your clock is wrong Zcash will not work properly."); } - SetMiscWarning(strMessage); + SetMiscWarning(strMessage, GetTime()); LogPrintf("*** %s\n", strMessage); uiInterface.ThreadSafeMessageBox(strMessage, "", CClientUIInterface::MSG_WARNING); } diff --git a/src/warnings.cpp b/src/warnings.cpp index 80d7f2755..9a52e488c 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -12,19 +12,24 @@ CCriticalSection cs_warnings; std::string strMiscWarning; +int64_t timestampWarning; bool fLargeWorkForkFound = false; bool fLargeWorkInvalidChainFound = false; -void SetMiscWarning(const std::string& strWarning) +void SetMiscWarning(const std::string& strWarning, int64_t timestamp) { LOCK(cs_warnings); strMiscWarning = strWarning; + timestampWarning = timestamp; } -std::string GetMiscWarning() +std::pair GetMiscWarning() { LOCK(cs_warnings); - return strMiscWarning; + std::pair misc; + misc.first = timestampWarning; + misc.second = strMiscWarning; + return misc; } void SetfLargeWorkForkFound(bool flag) @@ -51,36 +56,39 @@ bool GetfLargeWorkInvalidChainFound() return fLargeWorkInvalidChainFound; } -std::string GetWarnings(const std::string& strFor) +std::pair GetWarnings(const std::string& strFor) { + std::pair rpc; + std::pair statusbar; + rpc.first = GetTime(); + statusbar.first = GetTime(); int nPriority = 0; - std::string strStatusBar; - std::string strRPC; LOCK(cs_warnings); if (!CLIENT_VERSION_IS_RELEASE) - strStatusBar = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); + statusbar.second = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); if (GetBoolArg("-testsafemode", DEFAULT_TESTSAFEMODE)) - strStatusBar = strRPC = "testsafemode enabled"; + statusbar.second = rpc.second = "testsafemode enabled"; // Misc warnings like out of disk space and clock is wrong if (strMiscWarning != "") { nPriority = 1000; - strStatusBar = strMiscWarning; + statusbar.first = timestampWarning; + statusbar.second = strMiscWarning; } if (fLargeWorkForkFound) { nPriority = 2000; - strStatusBar = strRPC = _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."); + statusbar.second = rpc.second = _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."); } else if (fLargeWorkInvalidChainFound) { nPriority = 2000; - strStatusBar = strRPC = _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."); + statusbar.second = rpc.second = _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."); } // Alerts @@ -92,18 +100,18 @@ std::string GetWarnings(const std::string& strFor) if (alert.AppliesToMe() && alert.nPriority > nPriority) { nPriority = alert.nPriority; - strStatusBar = alert.strStatusBar; + statusbar.second = alert.strStatusBar; if (alert.nPriority >= ALERT_PRIORITY_SAFE_MODE) { - strRPC = alert.strRPCError; + rpc.second = alert.strRPCError; } } } } if (strFor == "statusbar") - return strStatusBar; + return statusbar; else if (strFor == "rpc") - return strRPC; + return rpc; assert(!"GetWarnings(): invalid parameter"); - return "error"; + return std::make_pair(0, "error"); } diff --git a/src/warnings.h b/src/warnings.h index 220b49504..ddbe3cf41 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -9,13 +9,13 @@ #include #include -void SetMiscWarning(const std::string& strWarning); -std::string GetMiscWarning(); +void SetMiscWarning(const std::string& strWarning, int64_t timestamp); +std::pair GetMiscWarning(); void SetfLargeWorkForkFound(bool flag); bool GetfLargeWorkForkFound(); void SetfLargeWorkInvalidChainFound(bool flag); bool GetfLargeWorkInvalidChainFound(); -std::string GetWarnings(const std::string& strFor); +std::pair GetWarnings(const std::string& strFor); static const bool DEFAULT_TESTSAFEMODE = false; From 1f06621e34741abab7dcbb5a2cb6927b063af7ad Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Mon, 15 Jun 2020 19:27:47 -0300 Subject: [PATCH 064/725] add myblockhash parameter to getrawtransaction Co-Authored-By: kallewoof --- qa/rpc-tests/rawtransactions.py | 26 ++++++++++- src/main.cpp | 83 +++++++++++++++++---------------- src/main.h | 2 +- src/rpc/rawtransaction.cpp | 53 +++++++++++++++++---- 4 files changed, 112 insertions(+), 52 deletions(-) diff --git a/qa/rpc-tests/rawtransactions.py b/qa/rpc-tests/rawtransactions.py index ec2be10ee..ba136fe01 100755 --- a/qa/rpc-tests/rawtransactions.py +++ b/qa/rpc-tests/rawtransactions.py @@ -11,7 +11,7 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.authproxy import JSONRPCException from test_framework.util import assert_equal, initialize_chain_clean, \ - start_nodes, connect_nodes_bi + start_nodes, connect_nodes_bi, assert_raises from decimal import Decimal @@ -68,6 +68,30 @@ class RawTransactionsTest(BitcoinTestFramework): assert_equal("Missing inputs" in errorString, True); + ##################################### + # getrawtransaction with block hash # + ##################################### + + # make a tx by sending then generate 2 blocks; block1 has the tx in it + tx = self.nodes[2].sendtoaddress(self.nodes[1].getnewaddress(), 1) + block1, block2 = self.nodes[2].generate(2) + self.sync_all() + # We should be able to get the raw transaction by providing the correct block + gottx = self.nodes[0].getrawtransaction(tx, 1, block1) + assert_equal(gottx['txid'], tx) + assert_equal(gottx['in_active_chain'], True) + # We should not have the 'in_active_chain' flag when we don't provide a block + gottx = self.nodes[0].getrawtransaction(tx, 1) + assert_equal(gottx['txid'], tx) + assert 'in_active_chain' not in gottx + # We should not get the tx if we provide an unrelated block + assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, tx, 1, block2) + # An invalid block hash should raise errors + assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, tx, 1, True) + assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, tx, 1, "foobar") + assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, tx, 1, "abcd1234") + assert_raises(JSONRPCException, self.nodes[0].getrawtransaction, tx, 1, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + ######################### # RAW TX MULTISIG TESTS # ######################### diff --git a/src/main.cpp b/src/main.cpp index d11ca5bae..8f57ad9db 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1648,49 +1648,57 @@ bool GetAddressUnspent(const uint160& addressHash, int type, return true; } -/** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ -bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) +/** + * Return transaction in txOut, and if it was found inside a block, its hash is placed in hashBlock. + * If blockIndex is provided, the transaction is fetched from the corresponding block. + */ +bool GetTransaction(const uint256& hash, CTransaction& txOut, const Consensus::Params& consensusParams, uint256& hashBlock, bool fAllowSlow, CBlockIndex* blockIndex) { - CBlockIndex *pindexSlow = NULL; + CBlockIndex* pindexSlow = blockIndex; LOCK(cs_main); - if (mempool.lookup(hash, txOut)) - { - return true; - } - - if (fTxIndex) { - CDiskTxPos postx; - if (pblocktree->ReadTxIndex(hash, postx)) { - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); - if (file.IsNull()) - return error("%s: OpenBlockFile failed", __func__); - CBlockHeader header; - try { - file >> header; - fseek(file.Get(), postx.nTxOffset, SEEK_CUR); - file >> txOut; - } catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); - } - hashBlock = header.GetHash(); - if (txOut.GetHash() != hash) - return error("%s: txid mismatch", __func__); + if (!blockIndex) { + if (mempool.lookup(hash, txOut)) + { return true; } - } - if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - int nHeight = -1; - { - CCoinsViewCache &view = *pcoinsTip; - const CCoins* coins = view.AccessCoins(hash); - if (coins) - nHeight = coins->nHeight; + if (fTxIndex) { + CDiskTxPos postx; + if (pblocktree->ReadTxIndex(hash, postx)) { + CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) + return error("%s: OpenBlockFile failed", __func__); + CBlockHeader header; + try { + file >> header; + fseek(file.Get(), postx.nTxOffset, SEEK_CUR); + file >> txOut; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + hashBlock = header.GetHash(); + if (txOut.GetHash() != hash) + return error("%s: txid mismatch", __func__); + return true; + } + + // transaction not found in index, nothing more can be done + return false; + } + + if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it + int nHeight = -1; + { + CCoinsViewCache &view = *pcoinsTip; + const CCoins* coins = view.AccessCoins(hash); + if (coins) + nHeight = coins->nHeight; + } + if (nHeight > 0) + pindexSlow = chainActive[nHeight]; } - if (nHeight > 0) - pindexSlow = chainActive[nHeight]; } if (pindexSlow) { @@ -1709,11 +1717,6 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::P return false; } - - - - - ////////////////////////////////////////////////////////////////////////////// // // CBlock and CBlockIndex diff --git a/src/main.h b/src/main.h index 193d06c56..4761d521a 100644 --- a/src/main.h +++ b/src/main.h @@ -252,7 +252,7 @@ bool IsInitialBlockDownload(const CChainParams& chainParams); /** Format a string that describes several potential problems detected by the core */ std::string GetWarnings(const std::string& strFor); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ -bool GetTransaction(const uint256 &hash, CTransaction &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); +bool GetTransaction(const uint256& hash, CTransaction& tx, const Consensus::Params& params, uint256& hashBlock, bool fAllowSlow = false, CBlockIndex* blockIndex = nullptr); /** Find the best known block, and make it the tip of the block chain */ bool ActivateBestChain(CValidationState& state, const CChainParams& chainparams, const CBlock* pblock = NULL); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 45c3adcfc..8765c23e5 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -251,12 +251,14 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) UniValue getrawtransaction(const UniValue& params, bool fHelp) { - if (fHelp || params.size() < 1 || params.size() > 2) + if (fHelp || params.size() < 1 || params.size() > 3) throw runtime_error( - "getrawtransaction \"txid\" ( verbose )\n" - "\nNOTE: By default this function only works sometimes. This is when the tx is in the mempool\n" - "or there is an unspent output in the utxo for this transaction. To make it always work,\n" - "you need to maintain a transaction index, using the -txindex command line option.\n" + "getrawtransaction \"txid\" ( verbose \"blockhash\" )\n" + "\nNOTE: By default this function only works for mempool transactions. If the -txindex option is\n" + "enabled, it also works for blockchain transactions. If the block which contains the transaction\n" + "is known, its hash can be provided even for nodes without -txindex. Note that if a blockhash is\n" + "provided, only that block will be searched and if the transaction is in the mempool or other\n" + "blocks, or if this node does not have the given block available, the transaction will not be found.\n" "\nReturn the raw transaction data.\n" "\nIf verbose=0, returns a string that is serialized, hex-encoded data for 'txid'.\n" "If verbose is non-zero, returns an Object with information about 'txid'.\n" @@ -264,12 +266,14 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) "\nArguments:\n" "1. \"txid\" (string, required) The transaction id\n" "2. verbose (numeric, optional, default=0) If 0, return a string, other return a json object\n" + "3. \"blockhash\" (string, optional) The block in which to look for the transaction\n" "\nResult (if verbose is not set or set to 0):\n" "\"data\" (string) The serialized, hex-encoded data for 'txid'\n" "\nResult (if verbose > 0):\n" "{\n" + " \"in_active_chain\": b, (bool) Whether specified block is in the active chain or not (only present with explicit \"blockhash\" argument)\n" " \"hex\" : \"data\", (string) The serialized, hex-encoded data for 'txid'\n" " \"txid\" : \"id\", (string) The transaction id (same as provided)\n" " \"version\" : n, (numeric) The version\n" @@ -341,20 +345,48 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) + HelpExampleCli("getrawtransaction", "\"mytxid\"") + HelpExampleCli("getrawtransaction", "\"mytxid\" 1") + HelpExampleRpc("getrawtransaction", "\"mytxid\", 1") + + HelpExampleCli("getrawtransaction", "\"mytxid\" 0 \"myblockhash\"") + + HelpExampleCli("getrawtransaction", "\"mytxid\" 1 \"myblockhash\"") ); + LOCK(cs_main); + + bool in_active_chain = true; uint256 hash = ParseHashV(params[0], "parameter 1"); + CBlockIndex* blockindex = nullptr; bool fVerbose = false; if (params.size() > 1) fVerbose = (params[1].get_int() != 0); - LOCK(cs_main); + if (params.size() > 2) { + uint256 blockhash = ParseHashV(params[2], "parameter 3"); + if (!blockhash.IsNull()) { + BlockMap::iterator it = mapBlockIndex.find(blockhash); + if (it == mapBlockIndex.end()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block hash not found"); + } + blockindex = it->second; + in_active_chain = chainActive.Contains(blockindex); + } + } CTransaction tx; - uint256 hashBlock; - if (!GetTransaction(hash, tx, Params().GetConsensus(), hashBlock, true)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available about transaction"); + uint256 hash_block; + if (!GetTransaction(hash, tx, Params().GetConsensus(), hash_block, true, blockindex)) { + std::string errmsg; + if (blockindex) { + if (!(blockindex->nStatus & BLOCK_HAVE_DATA)) { + throw JSONRPCError(RPC_MISC_ERROR, "Block not available"); + } + errmsg = "No such transaction found in the provided block"; + } else { + errmsg = fTxIndex + ? "No such mempool or blockchain transaction" + : "No such mempool transaction. Use -txindex to enable blockchain transaction queries"; + } + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errmsg + ". Use gettransaction for wallet transactions."); + } string strHex = EncodeHexTx(tx); @@ -362,8 +394,9 @@ UniValue getrawtransaction(const UniValue& params, bool fHelp) return strHex; UniValue result(UniValue::VOBJ); + if (blockindex) result.pushKV("in_active_chain", in_active_chain); result.pushKV("hex", strHex); - TxToJSON(tx, hashBlock, result); + TxToJSON(tx, hash_block, result); return result; } From 6581970d59210c0209d9d616769cded933dbae9c Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Wed, 17 Jun 2020 10:59:48 -0600 Subject: [PATCH 065/725] Add implementations of PRF_expand calls that obtain esk and rcm. --- src/zcash/prf.cpp | 16 ++++++++++++++++ src/zcash/prf.h | 2 ++ 2 files changed, 18 insertions(+) diff --git a/src/zcash/prf.cpp b/src/zcash/prf.cpp index 2491de83e..1aa8142d8 100644 --- a/src/zcash/prf.cpp +++ b/src/zcash/prf.cpp @@ -24,6 +24,22 @@ std::array PRF_expand(const uint256& sk, unsigned char t) return res; } +uint256 PRF_rcm(const uint256& rseed) +{ + uint256 rcm; + auto tmp = PRF_expand(rseed, 4); + librustzcash_to_scalar(tmp.data(), rcm.begin()); + return rcm; +} + +uint256 PRF_esk(const uint256& rseed) +{ + uint256 esk; + auto tmp = PRF_expand(rseed, 5); + librustzcash_to_scalar(tmp.data(), esk.begin()); + return esk; +} + uint256 PRF_ask(const uint256& sk) { uint256 ask; diff --git a/src/zcash/prf.h b/src/zcash/prf.h index f666cfa23..b9256769a 100644 --- a/src/zcash/prf.h +++ b/src/zcash/prf.h @@ -22,6 +22,8 @@ uint256 PRF_rho(const uint252& phi, size_t i0, const uint256& h_sig); uint256 PRF_ask(const uint256& sk); uint256 PRF_nsk(const uint256& sk); uint256 PRF_ovk(const uint256& sk); +uint256 PRF_rcm(const uint256& rseed); +uint256 PRF_esk(const uint256& rseed); std::array default_diversifier(const uint256& sk); From 1c2c6be872699e9be6d7f91aab1a5c3f9d329375 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Wed, 17 Jun 2020 11:36:35 -0600 Subject: [PATCH 066/725] Remove bare SaplingNote constructor. --- src/wallet/test/rpc_wallet_tests.cpp | 2 +- src/zcash/Note.hpp | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/wallet/test/rpc_wallet_tests.cpp b/src/wallet/test/rpc_wallet_tests.cpp index 621d7314b..6707c065f 100644 --- a/src/wallet/test/rpc_wallet_tests.cpp +++ b/src/wallet/test/rpc_wallet_tests.cpp @@ -1837,7 +1837,7 @@ BOOST_AUTO_TEST_CASE(rpc_z_mergetoaddress_parameters) std::vector sproutNoteInputs = {MergeToAddressInputSproutNote{JSOutPoint(), SproutNote(), 0, SproutSpendingKey()}}; std::vector saplingNoteInputs = - {MergeToAddressInputSaplingNote{SaplingOutPoint(), SaplingNote(), 0, SaplingExpandedSpendingKey()}}; + {MergeToAddressInputSaplingNote{SaplingOutPoint(), SaplingNote({}, uint256(), 0, uint256()), 0, SaplingExpandedSpendingKey()}}; // Sprout and Sapling inputs -> throw try { diff --git a/src/zcash/Note.hpp b/src/zcash/Note.hpp index dee17bfe4..357e27e48 100644 --- a/src/zcash/Note.hpp +++ b/src/zcash/Note.hpp @@ -50,8 +50,6 @@ public: SaplingNote(diversifier_t d, uint256 pk_d, uint64_t value, uint256 r) : BaseNote(value), d(d), pk_d(pk_d), r(r) {} - SaplingNote() {}; - SaplingNote(const SaplingPaymentAddress &address, uint64_t value); virtual ~SaplingNote() {}; From e71d2fba9da0675ecb0df565d7d1e81739a3a541 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Wed, 17 Jun 2020 13:18:20 -0600 Subject: [PATCH 067/725] Add a getter method to obtain rcm from a Sapling note plaintext. --- src/gtest/test_noteencryption.cpp | 4 ++-- src/zcash/Note.cpp | 14 ++++++++++---- src/zcash/Note.hpp | 7 +++++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/gtest/test_noteencryption.cpp b/src/gtest/test_noteencryption.cpp index d66580ac9..fca0ae1a8 100644 --- a/src/gtest/test_noteencryption.cpp +++ b/src/gtest/test_noteencryption.cpp @@ -78,7 +78,7 @@ TEST(noteencryption, NotePlaintext) ASSERT_TRUE(bar.value() == pt.value()); ASSERT_TRUE(bar.memo() == pt.memo()); ASSERT_TRUE(bar.d == pt.d); - ASSERT_TRUE(bar.rcm == pt.rcm); + ASSERT_TRUE(bar.rcm() == pt.rcm()); auto foobar = bar.note(ivk); @@ -155,7 +155,7 @@ TEST(noteencryption, NotePlaintext) ASSERT_TRUE(bar.value() == pt.value()); ASSERT_TRUE(bar.memo() == pt.memo()); ASSERT_TRUE(bar.d == pt.d); - ASSERT_TRUE(bar.rcm == pt.rcm); + ASSERT_TRUE(bar.rcm() == pt.rcm()); } TEST(noteencryption, SaplingApi) diff --git a/src/zcash/Note.cpp b/src/zcash/Note.cpp index 5f8b439fb..a315f428a 100644 --- a/src/zcash/Note.cpp +++ b/src/zcash/Note.cpp @@ -145,7 +145,7 @@ SaplingNotePlaintext::SaplingNotePlaintext( std::array memo) : BaseNotePlaintext(note, memo) { d = note.d; - rcm = note.r; + rseed = note.r; } @@ -153,7 +153,7 @@ boost::optional SaplingNotePlaintext::note(const SaplingIncomingVie { auto addr = ivk.address(d); if (addr) { - return SaplingNote(d, addr.get().pk_d, value_, rcm); + return SaplingNote(d, addr.get().pk_d, value_, rcm()); } else { return boost::none; } @@ -221,11 +221,12 @@ boost::optional SaplingNotePlaintext::decrypt( } uint256 cmu_expected; + uint256 rcm = ret.rcm(); if (!librustzcash_sapling_compute_cm( ret.d.data(), pk_d.begin(), ret.value(), - ret.rcm.begin(), + rcm.begin(), cmu_expected.begin() )) { @@ -266,11 +267,12 @@ boost::optional SaplingNotePlaintext::decrypt( } uint256 cmu_expected; + uint256 rcm = ret.rcm(); if (!librustzcash_sapling_compute_cm( ret.d.data(), pk_d.begin(), ret.value(), - ret.rcm.begin(), + rcm.begin(), cmu_expected.begin() )) { @@ -325,3 +327,7 @@ SaplingOutCiphertext SaplingOutgoingPlaintext::encrypt( return enc.encrypt_to_ourselves(ovk, cv, cm, pt); } + +uint256 SaplingNotePlaintext::rcm() const { + return rseed; +} diff --git a/src/zcash/Note.hpp b/src/zcash/Note.hpp index 357e27e48..a7c3023c0 100644 --- a/src/zcash/Note.hpp +++ b/src/zcash/Note.hpp @@ -117,9 +117,10 @@ public: typedef std::pair SaplingNotePlaintextEncryptionResult; class SaplingNotePlaintext : public BaseNotePlaintext { +private: + uint256 rseed; public: diversifier_t d; - uint256 rcm; SaplingNotePlaintext() {} @@ -157,11 +158,13 @@ public: READWRITE(d); // 11 bytes READWRITE(value_); // 8 bytes - READWRITE(rcm); // 32 bytes + READWRITE(rseed); // 32 bytes READWRITE(memo_); // 512 bytes } boost::optional encrypt(const uint256& pk_d) const; + + uint256 rcm() const; }; class SaplingOutgoingPlaintext From 8770a5c53234831f9d814cb63f1e836db6478a44 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Wed, 17 Jun 2020 14:53:12 -0600 Subject: [PATCH 068/725] Add support for receiving v2 Sapling note plaintexts. Co-authored by Ying Tong (yingtong@electriccoin.co) --- src/gtest/test_checktransaction.cpp | 4 +- src/gtest/test_noteencryption.cpp | 12 ++-- src/gtest/test_sapling_note.cpp | 8 +-- src/gtest/test_transaction_builder.cpp | 2 +- src/miner.cpp | 2 +- src/transaction_builder.cpp | 8 ++- src/utiltest.cpp | 2 +- src/wallet/gtest/test_wallet.cpp | 6 +- src/zcash/Note.cpp | 85 +++++++++++++++++++++++--- src/zcash/Note.hpp | 26 +++++--- src/zcash/NoteEncryption.cpp | 15 +++-- src/zcash/NoteEncryption.hpp | 2 +- src/zcbenchmarks.cpp | 10 +-- 13 files changed, 134 insertions(+), 48 deletions(-) diff --git a/src/gtest/test_checktransaction.cpp b/src/gtest/test_checktransaction.cpp index 9b0ff0ae6..233dcb03c 100644 --- a/src/gtest/test_checktransaction.cpp +++ b/src/gtest/test_checktransaction.cpp @@ -1134,7 +1134,7 @@ TEST(CheckTransaction, HeartwoodAcceptsShieldedCoinbase) { uint256 ovk; auto note = libzcash::SaplingNote( - libzcash::SaplingSpendingKey::random().default_address(), CAmount(123456)); + libzcash::SaplingSpendingKey::random().default_address(), CAmount(123456), 0x01); auto output = OutputDescriptionInfo(ovk, note, {{0xF6}}); auto ctx = librustzcash_sapling_proving_ctx_init(); @@ -1217,7 +1217,7 @@ TEST(CheckTransaction, HeartwoodEnforcesSaplingRulesOnShieldedCoinbase) { uint256 ovk; auto note = libzcash::SaplingNote( - libzcash::SaplingSpendingKey::random().default_address(), CAmount(123456)); + libzcash::SaplingSpendingKey::random().default_address(), CAmount(123456), 0x01); auto output = OutputDescriptionInfo(ovk, note, {{0xF6}}); CMutableTransaction mtx = GetValidTransaction(); diff --git a/src/gtest/test_noteencryption.cpp b/src/gtest/test_noteencryption.cpp index fca0ae1a8..083bdeba1 100644 --- a/src/gtest/test_noteencryption.cpp +++ b/src/gtest/test_noteencryption.cpp @@ -34,7 +34,7 @@ TEST(noteencryption, NotePlaintext) memo[i] = (unsigned char) i; } - SaplingNote note(addr, 39393); + SaplingNote note(addr, 39393, 0x01); auto cmu_opt = note.cmu(); if (!cmu_opt) { FAIL(); @@ -91,7 +91,7 @@ TEST(noteencryption, NotePlaintext) ASSERT_TRUE(note.value() == new_note.value()); ASSERT_TRUE(note.d == new_note.d); ASSERT_TRUE(note.pk_d == new_note.pk_d); - ASSERT_TRUE(note.r == new_note.r); + ASSERT_TRUE(note.rcm() == new_note.rcm()); ASSERT_TRUE(note.cmu() == new_note.cmu()); SaplingOutgoingPlaintext out_pt; @@ -183,10 +183,10 @@ TEST(noteencryption, SaplingApi) } // Invalid diversifier - ASSERT_EQ(boost::none, SaplingNoteEncryption::FromDiversifier({1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0})); + ASSERT_EQ(boost::none, SaplingNoteEncryption::FromDiversifier({1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, boost::none)); // Encrypt to pk_1 - auto enc = *SaplingNoteEncryption::FromDiversifier(pk_1.d); + auto enc = *SaplingNoteEncryption::FromDiversifier(pk_1.d, boost::none); auto ciphertext_1 = *enc.encrypt_to_recipient( pk_1.pk_d, message @@ -208,7 +208,7 @@ TEST(noteencryption, SaplingApi) ); // Encrypt to pk_2 - enc = *SaplingNoteEncryption::FromDiversifier(pk_2.d); + enc = *SaplingNoteEncryption::FromDiversifier(pk_2.d, boost::none); auto ciphertext_2 = *enc.encrypt_to_recipient( pk_2.pk_d, message @@ -226,7 +226,7 @@ TEST(noteencryption, SaplingApi) // Test nonce-reuse resistance of API { - auto tmp_enc = *SaplingNoteEncryption::FromDiversifier(pk_1.d); + auto tmp_enc = *SaplingNoteEncryption::FromDiversifier(pk_1.d, boost::none); tmp_enc.encrypt_to_recipient( pk_1.pk_d, diff --git a/src/gtest/test_sapling_note.cpp b/src/gtest/test_sapling_note.cpp index b6846217a..b1da05da3 100644 --- a/src/gtest/test_sapling_note.cpp +++ b/src/gtest/test_sapling_note.cpp @@ -57,16 +57,16 @@ TEST(SaplingNote, Random) { // Test creating random notes using the same spending key auto address = SaplingSpendingKey::random().default_address(); - SaplingNote note1(address, GetRand(MAX_MONEY)); - SaplingNote note2(address, GetRand(MAX_MONEY)); + SaplingNote note1(address, GetRand(MAX_MONEY), 0x01); + SaplingNote note2(address, GetRand(MAX_MONEY), 0x01); ASSERT_EQ(note1.d, note2.d); ASSERT_EQ(note1.pk_d, note2.pk_d); ASSERT_NE(note1.value(), note2.value()); - ASSERT_NE(note1.r, note2.r); + ASSERT_NE(note1.rcm(), note2.rcm()); // Test diversifier and pk_d are not the same for different spending keys - SaplingNote note3(SaplingSpendingKey::random().default_address(), GetRand(MAX_MONEY)); + SaplingNote note3(SaplingSpendingKey::random().default_address(), GetRand(MAX_MONEY), 0x01); ASSERT_NE(note1.d, note3.d); ASSERT_NE(note1.pk_d, note3.pk_d); } diff --git a/src/gtest/test_transaction_builder.cpp b/src/gtest/test_transaction_builder.cpp index d97d0287e..3cc382265 100644 --- a/src/gtest/test_transaction_builder.cpp +++ b/src/gtest/test_transaction_builder.cpp @@ -482,7 +482,7 @@ TEST(TransactionBuilder, CheckSaplingTxVersion) } // Cannot add Sapling spends to a non-Sapling transaction - libzcash::SaplingNote note(pk, 50000); + libzcash::SaplingNote note(pk, 50000, 0x01); SaplingMerkleTree tree; try { builder.AddSaplingSpend(expsk, note, uint256(), tree.witness()); diff --git a/src/miner.cpp b/src/miner.cpp index 89f52175a..489e09dbc 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -157,7 +157,7 @@ public: mtx.valueBalance = -value; uint256 ovk; - auto note = libzcash::SaplingNote(pa, value); + auto note = libzcash::SaplingNote(pa, value, 0x01); // TODO auto output = OutputDescriptionInfo(ovk, note, {{0xF6}}); auto ctx = librustzcash_sapling_proving_ctx_init(); diff --git a/src/transaction_builder.cpp b/src/transaction_builder.cpp index 811db2740..07b9087ff 100644 --- a/src/transaction_builder.cpp +++ b/src/transaction_builder.cpp @@ -43,11 +43,12 @@ boost::optional OutputDescriptionInfo::Build(void* ctx) { std::vector addressBytes(ss.begin(), ss.end()); OutputDescription odesc; + uint256 rcm = this->note.rcm(); if (!librustzcash_sapling_output_proof( ctx, encryptor.get_esk().begin(), addressBytes.data(), - this->note.r.begin(), + rcm.begin(), this->note.value(), odesc.cv.begin(), odesc.zkproof.begin())) { @@ -161,7 +162,7 @@ void TransactionBuilder::AddSaplingOutput( throw std::runtime_error("TransactionBuilder cannot add Sapling output to pre-Sapling transaction"); } - auto note = libzcash::SaplingNote(to, value); + auto note = libzcash::SaplingNote(to, value, 0x01); // TODO outputs.emplace_back(ovk, note, memo); mtx.valueBalance -= value; } @@ -324,12 +325,13 @@ TransactionBuilderResult TransactionBuilder::Build() std::vector witness(ss.begin(), ss.end()); SpendDescription sdesc; + uint256 rcm = spend.note.rcm(); if (!librustzcash_sapling_spend_proof( ctx, spend.expsk.full_viewing_key().ak.begin(), spend.expsk.nsk.begin(), spend.note.d.data(), - spend.note.r.begin(), + rcm.begin(), spend.alpha.begin(), spend.note.value(), spend.anchor.begin(), diff --git a/src/utiltest.cpp b/src/utiltest.cpp index bcb47abeb..bb75a9504 100644 --- a/src/utiltest.cpp +++ b/src/utiltest.cpp @@ -289,7 +289,7 @@ CKey AddTestCKeyToKeyStore(CBasicKeyStore& keyStore) { TestSaplingNote GetTestSaplingNote(const libzcash::SaplingPaymentAddress& pa, CAmount value) { // Generate dummy Sapling note - libzcash::SaplingNote note(pa, value); + libzcash::SaplingNote note(pa, value, 0x01); uint256 cm = note.cmu().get(); SaplingMerkleTree tree; tree.append(cm); diff --git a/src/wallet/gtest/test_wallet.cpp b/src/wallet/gtest/test_wallet.cpp index 51f233c7d..18865cbcd 100644 --- a/src/wallet/gtest/test_wallet.cpp +++ b/src/wallet/gtest/test_wallet.cpp @@ -387,7 +387,7 @@ TEST(WalletTests, SetSaplingNoteAddrsInCWalletTx) { auto ivk = fvk.in_viewing_key(); auto pk = sk.DefaultAddress(); - libzcash::SaplingNote note(pk, 50000); + libzcash::SaplingNote note(pk, 50000, 0x01); auto cm = note.cmu().get(); SaplingMerkleTree tree; tree.append(cm); @@ -654,7 +654,7 @@ TEST(WalletTests, GetConflictedSaplingNotes) { ASSERT_TRUE(wallet.HaveSaplingSpendingKey(extfvk)); // Generate note A - libzcash::SaplingNote note(pk, 50000); + libzcash::SaplingNote note(pk, 50000, 0x01); auto cm = note.cmu().get(); SaplingMerkleTree saplingTree; saplingTree.append(cm); @@ -1018,7 +1018,7 @@ TEST(WalletTests, SpentSaplingNoteIsFromMe) { auto pk = sk.DefaultAddress(); // Generate Sapling note A - libzcash::SaplingNote note(pk, 50000); + libzcash::SaplingNote note(pk, 50000, 0x01); auto cm = note.cmu().get(); SaplingMerkleTree saplingTree; saplingTree.append(cm); diff --git a/src/zcash/Note.cpp b/src/zcash/Note.cpp index a315f428a..76a14f9e6 100644 --- a/src/zcash/Note.cpp +++ b/src/zcash/Note.cpp @@ -41,20 +41,31 @@ uint256 SproutNote::nullifier(const SproutSpendingKey& a_sk) const { } // Construct and populate Sapling note for a given payment address and value. -SaplingNote::SaplingNote(const SaplingPaymentAddress& address, const uint64_t value) : BaseNote(value) { +SaplingNote::SaplingNote( + const SaplingPaymentAddress& address, + const uint64_t value, + unsigned char _leadByte +) : BaseNote(value) { d = address.d; pk_d = address.pk_d; - librustzcash_sapling_generate_r(r.begin()); + leadByte = _leadByte; + if (leadByte == 0x02) { + // Per ZIP 212, the rseed field is 32 random bytes. + rseed = random_uint256(); + } else { + librustzcash_sapling_generate_r(rseed.begin()); + } } // Call librustzcash to compute the commitment boost::optional SaplingNote::cmu() const { uint256 result; + uint256 rcm_tmp = rcm(); if (!librustzcash_sapling_compute_cm( d.data(), pk_d.begin(), value(), - r.begin(), + rcm_tmp.begin(), result.begin() )) { @@ -71,11 +82,12 @@ boost::optional SaplingNote::nullifier(const SaplingFullViewingKey& vk, auto nk = vk.nk; uint256 result; + uint256 rcm_tmp = rcm(); if (!librustzcash_sapling_compute_nf( d.data(), pk_d.begin(), value(), - r.begin(), + rcm_tmp.begin(), ak.begin(), nk.begin(), position, @@ -145,7 +157,8 @@ SaplingNotePlaintext::SaplingNotePlaintext( std::array memo) : BaseNotePlaintext(note, memo) { d = note.d; - rseed = note.r; + rseed = note.rseed; + leadByte = note.leadByte; } @@ -153,7 +166,9 @@ boost::optional SaplingNotePlaintext::note(const SaplingIncomingVie { auto addr = ivk.address(d); if (addr) { - return SaplingNote(d, addr.get().pk_d, value_, rcm()); + auto tmp = SaplingNote(d, addr.get().pk_d, value_, rseed); + tmp.leadByte = leadByte; + return tmp; } else { return boost::none; } @@ -237,6 +252,19 @@ boost::optional SaplingNotePlaintext::decrypt( return boost::none; } + if (ret.leadByte == 0x02) { + // ZIP 212: Check that epk is consistent to prevent against linkability + // attacks without relying on the soundness of the SNARK. + uint256 expected_epk; + uint256 esk = ret.generate_esk(); + if (!librustzcash_sapling_ka_derivepublic(ret.d.data(), esk.begin(), expected_epk.begin())) { + return boost::none; + } + if (expected_epk != epk) { + return boost::none; + } + } + return ret; } @@ -283,13 +311,31 @@ boost::optional SaplingNotePlaintext::decrypt( return boost::none; } + if (ret.leadByte == 0x02) { + // ZIP 212: Check that epk is consistent to prevent against linkability + // attacks without relying on the soundness of the SNARK. + uint256 expected_epk; + if (!librustzcash_sapling_ka_derivepublic(ret.d.data(), esk.begin(), expected_epk.begin())) { + return boost::none; + } + if (expected_epk != epk) { + return boost::none; + } + + // ZIP 212: Additionally check that the esk provided to this function + // is consistent with the esk we can derive + if (esk != ret.generate_esk()) { + return boost::none; + } + } + return ret; } boost::optional SaplingNotePlaintext::encrypt(const uint256& pk_d) const { // Get the encryptor - auto sne = SaplingNoteEncryption::FromDiversifier(d); + auto sne = SaplingNoteEncryption::FromDiversifier(d, generate_esk()); if (!sne) { return boost::none; } @@ -329,5 +375,28 @@ SaplingOutCiphertext SaplingOutgoingPlaintext::encrypt( } uint256 SaplingNotePlaintext::rcm() const { - return rseed; + if (leadByte == 0x02) { + return PRF_rcm(rseed); + } else { + return rseed; + } +} + +uint256 SaplingNote::rcm() const { + if (leadByte == 0x02) { + return PRF_rcm(rseed); + } else { + return rseed; + } +} + +uint256 SaplingNotePlaintext::generate_esk() const { + if (leadByte == 0x02) { + return PRF_esk(rseed); + } else { + uint256 esk; + // Pick random esk + librustzcash_sapling_generate_r(esk.begin()); + return esk; + } } diff --git a/src/zcash/Note.hpp b/src/zcash/Note.hpp index a7c3023c0..a130267a7 100644 --- a/src/zcash/Note.hpp +++ b/src/zcash/Note.hpp @@ -42,20 +42,24 @@ public: class SaplingNote : public BaseNote { +private: + uint256 rseed; + friend class SaplingNotePlaintext; + unsigned char leadByte; public: diversifier_t d; uint256 pk_d; - uint256 r; - SaplingNote(diversifier_t d, uint256 pk_d, uint64_t value, uint256 r) - : BaseNote(value), d(d), pk_d(pk_d), r(r) {} + SaplingNote(diversifier_t d, uint256 pk_d, uint64_t value, uint256 rseed) + : BaseNote(value), d(d), pk_d(pk_d), rseed(rseed) {} - SaplingNote(const SaplingPaymentAddress &address, uint64_t value); + SaplingNote(const SaplingPaymentAddress &address, uint64_t value, unsigned char leadByte); virtual ~SaplingNote() {}; boost::optional cmu() const; boost::optional nullifier(const SaplingFullViewingKey &vk, const uint64_t position) const; + uint256 rcm() const; }; class BaseNotePlaintext { @@ -89,10 +93,10 @@ public: template inline void SerializationOp(Stream& s, Operation ser_action) { - unsigned char leadingByte = 0x00; - READWRITE(leadingByte); + unsigned char leadByte = 0x00; + READWRITE(leadByte); - if (leadingByte != 0x00) { + if (leadByte != 0x00) { throw std::ios_base::failure("lead byte of SproutNotePlaintext is not recognized"); } @@ -119,6 +123,7 @@ typedef std::pair SaplingNotePlaint class SaplingNotePlaintext : public BaseNotePlaintext { private: uint256 rseed; + unsigned char leadByte; public: diversifier_t d; @@ -149,10 +154,10 @@ public: template inline void SerializationOp(Stream& s, Operation ser_action) { - unsigned char leadingByte = 0x01; - READWRITE(leadingByte); + READWRITE(leadByte); - if (leadingByte != 0x01) { + if (leadByte != 0x01 && leadByte != 0x02) { + printf("leadByte: %x\n", leadByte); throw std::ios_base::failure("lead byte of SaplingNotePlaintext is not recognized"); } @@ -165,6 +170,7 @@ public: boost::optional encrypt(const uint256& pk_d) const; uint256 rcm() const; + uint256 generate_esk() const; }; class SaplingOutgoingPlaintext diff --git a/src/zcash/NoteEncryption.cpp b/src/zcash/NoteEncryption.cpp index 63e073265..4af47d42e 100644 --- a/src/zcash/NoteEncryption.cpp +++ b/src/zcash/NoteEncryption.cpp @@ -101,12 +101,19 @@ void KDF(unsigned char K[NOTEENCRYPTION_CIPHER_KEYSIZE], namespace libzcash { -boost::optional SaplingNoteEncryption::FromDiversifier(diversifier_t d) { +boost::optional SaplingNoteEncryption::FromDiversifier( + diversifier_t d, + boost::optional esk_provided +) +{ uint256 epk; uint256 esk; - - // Pick random esk - librustzcash_sapling_generate_r(esk.begin()); + if (esk_provided) { + esk = esk_provided.get(); + } else { + // Pick random esk + librustzcash_sapling_generate_r(esk.begin()); + } // Compute epk given the diversifier if (!librustzcash_sapling_ka_derivepublic(d.begin(), esk.begin(), epk.begin())) { diff --git a/src/zcash/NoteEncryption.hpp b/src/zcash/NoteEncryption.hpp index f6e692028..51b27be17 100644 --- a/src/zcash/NoteEncryption.hpp +++ b/src/zcash/NoteEncryption.hpp @@ -42,7 +42,7 @@ protected: public: - static boost::optional FromDiversifier(diversifier_t d); + static boost::optional FromDiversifier(diversifier_t d, boost::optional esk); boost::optional encrypt_to_recipient( const uint256 &pk_d, diff --git a/src/zcbenchmarks.cpp b/src/zcbenchmarks.cpp index cd728eceb..33378f2db 100644 --- a/src/zcbenchmarks.cpp +++ b/src/zcbenchmarks.cpp @@ -594,7 +594,7 @@ double benchmark_create_sapling_spend() auto sk = libzcash::SaplingSpendingKey::random(); auto expsk = sk.expanded_spending_key(); auto address = sk.default_address(); - SaplingNote note(address, GetRand(MAX_MONEY)); + SaplingNote note(address, GetRand(MAX_MONEY), 0x01); SaplingMerkleTree tree; auto maybe_cmu = note.cmu(); tree.append(maybe_cmu.get()); @@ -618,12 +618,13 @@ double benchmark_create_sapling_spend() timer_start(tv_start); SpendDescription sdesc; + uint256 rcm = note.rcm(); bool result = librustzcash_sapling_spend_proof( ctx, expsk.full_viewing_key().ak.begin(), expsk.nsk.begin(), note.d.data(), - note.r.begin(), + rcm.begin(), alpha.begin(), note.value(), anchor.begin(), @@ -646,7 +647,7 @@ double benchmark_create_sapling_output() auto address = sk.default_address(); std::array memo; - SaplingNote note(address, GetRand(MAX_MONEY)); + SaplingNote note(address, GetRand(MAX_MONEY), 0x01); libzcash::SaplingNotePlaintext notePlaintext(note, memo); auto res = notePlaintext.encrypt(note.pk_d); @@ -667,11 +668,12 @@ double benchmark_create_sapling_output() timer_start(tv_start); OutputDescription odesc; + uint256 rcm = note.rcm(); bool result = librustzcash_sapling_output_proof( ctx, encryptor.get_esk().begin(), addressBytes.data(), - note.r.begin(), + rcm.begin(), note.value(), odesc.cv.begin(), odesc.zkproof.begin()); From bf9baf2cb65d5553924d0a3ca3d83d5837af975f Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Wed, 17 Jun 2020 16:02:37 -0600 Subject: [PATCH 069/725] Change transaction builder and miner to use v2 Sapling note plaintexts after Canopy activates. Co-authored by Ying Tong (yingtong@electriccoin.co) --- src/miner.cpp | 6 +++++- src/transaction_builder.cpp | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/miner.cpp b/src/miner.cpp index 489e09dbc..ddcdbf2bd 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -157,7 +157,11 @@ public: mtx.valueBalance = -value; uint256 ovk; - auto note = libzcash::SaplingNote(pa, value, 0x01); // TODO + unsigned char leadByte = 0x01; + if (Params().GetConsensus().NetworkUpgradeActive(nHeight, Consensus::UPGRADE_CANOPY)) { + leadByte = 0x02; + } + auto note = libzcash::SaplingNote(pa, value, leadByte); auto output = OutputDescriptionInfo(ovk, note, {{0xF6}}); auto ctx = librustzcash_sapling_proving_ctx_init(); diff --git a/src/transaction_builder.cpp b/src/transaction_builder.cpp index 07b9087ff..7390932af 100644 --- a/src/transaction_builder.cpp +++ b/src/transaction_builder.cpp @@ -162,7 +162,11 @@ void TransactionBuilder::AddSaplingOutput( throw std::runtime_error("TransactionBuilder cannot add Sapling output to pre-Sapling transaction"); } - auto note = libzcash::SaplingNote(to, value, 0x01); // TODO + unsigned char leadByte = 0x01; + if (Params().GetConsensus().NetworkUpgradeActive(nHeight, Consensus::UPGRADE_CANOPY)) { + leadByte = 0x02; + } + auto note = libzcash::SaplingNote(to, value, leadByte); outputs.emplace_back(ovk, note, memo); mtx.valueBalance -= value; } From 3688f508d8645398aeb1e83d5f3a199715aace82 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Mon, 22 Jun 2020 15:11:31 -0600 Subject: [PATCH 070/725] Make ed25519-zebra available via librustzcash. --- Cargo.lock | 105 ++++++++++++++++++++- Cargo.toml | 2 + depends/packages/crate_bincode.mk | 15 +++ depends/packages/crate_curve25519_dalek.mk | 15 +++ depends/packages/crate_ed25519_zebra.mk | 15 +++ depends/packages/crate_hex.mk | 2 +- depends/packages/crate_hex2.mk | 15 +++ depends/packages/crate_serde.mk | 15 +++ depends/packages/crate_serde_derive.mk | 15 +++ depends/packages/crate_subtle.mk | 15 +++ depends/packages/crate_syn.mk | 4 +- depends/packages/crate_thiserror.mk | 15 +++ depends/packages/crate_thiserror_impl.mk | 15 +++ depends/packages/crate_zeroize.mk | 15 +++ depends/packages/packages.mk | 12 ++- src/gtest/test_consensus.cpp | 8 ++ src/rust/include/librustzcash.h | 7 ++ src/rust/src/rustzcash.rs | 39 ++++++++ 18 files changed, 321 insertions(+), 8 deletions(-) create mode 100644 depends/packages/crate_bincode.mk create mode 100644 depends/packages/crate_curve25519_dalek.mk create mode 100644 depends/packages/crate_ed25519_zebra.mk create mode 100644 depends/packages/crate_hex2.mk create mode 100644 depends/packages/crate_serde.mk create mode 100644 depends/packages/crate_serde_derive.mk create mode 100644 depends/packages/crate_subtle.mk create mode 100644 depends/packages/crate_thiserror.mk create mode 100644 depends/packages/crate_thiserror_impl.mk create mode 100644 depends/packages/crate_zeroize.mk diff --git a/Cargo.lock b/Cargo.lock index a9334fd0e..651752348 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,6 +82,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "bincode" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" +dependencies = [ + "byteorder", + "serde", +] + [[package]] name = "bit-vec" version = "0.4.4" @@ -261,6 +271,19 @@ dependencies = [ "crypto_api", ] +[[package]] +name = "curve25519-dalek" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5" +dependencies = [ + "byteorder", + "digest", + "rand_core", + "subtle", + "zeroize", +] + [[package]] name = "digest" version = "0.8.1" @@ -280,6 +303,20 @@ dependencies = [ "winapi", ] +[[package]] +name = "ed25519-zebra" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a76f15c88332faad36abb368aca89deb5cc4f440e5181c8848f8bdd049848f7b" +dependencies = [ + "curve25519-dalek", + "hex 0.4.2", + "rand_core", + "serde", + "sha2", + "thiserror", +] + [[package]] name = "fake-simd" version = "0.1.2" @@ -377,6 +414,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + [[package]] name = "lazy_static" version = "1.4.0" @@ -394,8 +437,10 @@ name = "librustzcash" version = "0.2.0" dependencies = [ "bellman", + "bincode", "blake2b_simd", "blake2s_simd", + "ed25519-zebra", "ff", "lazy_static", "libc", @@ -591,6 +636,26 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +[[package]] +name = "serde" +version = "1.0.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6135c78461981c79497158ef777264c51d9d0f4f3fc3a4d22b915900e42dac6a" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.113" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93c5eaa17d0954cb481cdcfffe9d84fcfa7a1a9f2349271e678677be4c26ae31" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha2" version = "0.8.0" @@ -604,16 +669,42 @@ dependencies = [ ] [[package]] -name = "syn" -version = "1.0.5" +name = "subtle" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66850e97125af79138385e9b88339cbcd037e3f28ceab8c5ad98e64f0f1f80bf" +checksum = "502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1" + +[[package]] +name = "syn" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238" dependencies = [ "proc-macro2", "quote", "unicode-xid", ] +[[package]] +name = "thiserror" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "typenum" version = "1.11.2" @@ -678,7 +769,7 @@ dependencies = [ "crypto_api_chachapoly", "ff", "fpe", - "hex", + "hex 0.3.2", "lazy_static", "log", "pairing", @@ -702,3 +793,9 @@ dependencies = [ "rand_core", "zcash_primitives", ] + +[[package]] +name = "zeroize" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8" diff --git a/Cargo.toml b/Cargo.toml index 663af314f..3406b28b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,8 @@ rand_core = "0.5.1" zcash_history = "0.2" zcash_primitives = "0.2" zcash_proofs = "0.2" +ed25519-zebra = "0.4.1" +bincode = "1.2.1" [profile.release] lto = true diff --git a/depends/packages/crate_bincode.mk b/depends/packages/crate_bincode.mk new file mode 100644 index 000000000..fa8d8cdb6 --- /dev/null +++ b/depends/packages/crate_bincode.mk @@ -0,0 +1,15 @@ +package=crate_bincode +$(package)_crate_name=bincode +$(package)_version=1.2.1 +$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) +$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate +$(package)_sha256_hash=5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf +$(package)_crate_versioned_name=$($(package)_crate_name) + +define $(package)_preprocess_cmds + $(call generate_crate_checksum,$(package)) +endef + +define $(package)_stage_cmds + $(call vendor_crate_source,$(package)) +endef diff --git a/depends/packages/crate_curve25519_dalek.mk b/depends/packages/crate_curve25519_dalek.mk new file mode 100644 index 000000000..23469d0fc --- /dev/null +++ b/depends/packages/crate_curve25519_dalek.mk @@ -0,0 +1,15 @@ +package=crate_curve25519_dalek +$(package)_crate_name=curve25519-dalek +$(package)_version=2.1.0 +$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) +$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate +$(package)_sha256_hash=5d85653f070353a16313d0046f173f70d1aadd5b42600a14de626f0dfb3473a5 +$(package)_crate_versioned_name=$($(package)_crate_name) + +define $(package)_preprocess_cmds + $(call generate_crate_checksum,$(package)) +endef + +define $(package)_stage_cmds + $(call vendor_crate_source,$(package)) +endef diff --git a/depends/packages/crate_ed25519_zebra.mk b/depends/packages/crate_ed25519_zebra.mk new file mode 100644 index 000000000..f1c12a2db --- /dev/null +++ b/depends/packages/crate_ed25519_zebra.mk @@ -0,0 +1,15 @@ +package=crate_ed25519_zebra +$(package)_crate_name=ed25519-zebra +$(package)_version=0.4.1 +$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) +$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate +$(package)_sha256_hash=a76f15c88332faad36abb368aca89deb5cc4f440e5181c8848f8bdd049848f7b +$(package)_crate_versioned_name=$($(package)_crate_name) + +define $(package)_preprocess_cmds + $(call generate_crate_checksum,$(package)) +endef + +define $(package)_stage_cmds + $(call vendor_crate_source,$(package)) +endef diff --git a/depends/packages/crate_hex.mk b/depends/packages/crate_hex.mk index f93417a30..c24120d05 100644 --- a/depends/packages/crate_hex.mk +++ b/depends/packages/crate_hex.mk @@ -4,7 +4,7 @@ $(package)_version=0.3.2 $(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) $(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate $(package)_sha256_hash=805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77 -$(package)_crate_versioned_name=$($(package)_crate_name) +$(package)_crate_versioned_name="$($(package)_crate_name) 0.3.2" define $(package)_preprocess_cmds $(call generate_crate_checksum,$(package)) diff --git a/depends/packages/crate_hex2.mk b/depends/packages/crate_hex2.mk new file mode 100644 index 000000000..c589bc4ec --- /dev/null +++ b/depends/packages/crate_hex2.mk @@ -0,0 +1,15 @@ +package=crate_hex2 +$(package)_crate_name=hex +$(package)_version=0.4.2 +$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) +$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate +$(package)_sha256_hash=644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35 +$(package)_crate_versioned_name="$($(package)_crate_name) 0.4.2" + +define $(package)_preprocess_cmds + $(call generate_crate_checksum,$(package)) +endef + +define $(package)_stage_cmds + $(call vendor_crate_source,$(package)) +endef diff --git a/depends/packages/crate_serde.mk b/depends/packages/crate_serde.mk new file mode 100644 index 000000000..944c7c948 --- /dev/null +++ b/depends/packages/crate_serde.mk @@ -0,0 +1,15 @@ +package=crate_serde +$(package)_crate_name=serde +$(package)_version=1.0.113 +$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) +$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate +$(package)_sha256_hash=6135c78461981c79497158ef777264c51d9d0f4f3fc3a4d22b915900e42dac6a +$(package)_crate_versioned_name=$($(package)_crate_name) + +define $(package)_preprocess_cmds + $(call generate_crate_checksum,$(package)) +endef + +define $(package)_stage_cmds + $(call vendor_crate_source,$(package)) +endef diff --git a/depends/packages/crate_serde_derive.mk b/depends/packages/crate_serde_derive.mk new file mode 100644 index 000000000..a61dc3c76 --- /dev/null +++ b/depends/packages/crate_serde_derive.mk @@ -0,0 +1,15 @@ +package=crate_serde_derive +$(package)_crate_name=serde_derive +$(package)_version=1.0.113 +$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) +$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate +$(package)_sha256_hash=93c5eaa17d0954cb481cdcfffe9d84fcfa7a1a9f2349271e678677be4c26ae31 +$(package)_crate_versioned_name=$($(package)_crate_name) + +define $(package)_preprocess_cmds + $(call generate_crate_checksum,$(package)) +endef + +define $(package)_stage_cmds + $(call vendor_crate_source,$(package)) +endef diff --git a/depends/packages/crate_subtle.mk b/depends/packages/crate_subtle.mk new file mode 100644 index 000000000..bd43b168c --- /dev/null +++ b/depends/packages/crate_subtle.mk @@ -0,0 +1,15 @@ +package=crate_subtle +$(package)_crate_name=subtle +$(package)_version=2.2.3 +$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) +$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate +$(package)_sha256_hash=502d53007c02d7605a05df1c1a73ee436952781653da5d0bf57ad608f66932c1 +$(package)_crate_versioned_name=$($(package)_crate_name) + +define $(package)_preprocess_cmds + $(call generate_crate_checksum,$(package)) +endef + +define $(package)_stage_cmds + $(call vendor_crate_source,$(package)) +endef diff --git a/depends/packages/crate_syn.mk b/depends/packages/crate_syn.mk index 37593b2e1..557e8db3a 100644 --- a/depends/packages/crate_syn.mk +++ b/depends/packages/crate_syn.mk @@ -1,9 +1,9 @@ package=crate_syn $(package)_crate_name=syn -$(package)_version=1.0.5 +$(package)_version=1.0.11 $(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) $(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate -$(package)_sha256_hash=66850e97125af79138385e9b88339cbcd037e3f28ceab8c5ad98e64f0f1f80bf +$(package)_sha256_hash=dff0acdb207ae2fe6d5976617f887eb1e35a2ba52c13c7234c790960cdad9238 $(package)_crate_versioned_name=$($(package)_crate_name) define $(package)_preprocess_cmds diff --git a/depends/packages/crate_thiserror.mk b/depends/packages/crate_thiserror.mk new file mode 100644 index 000000000..15234384b --- /dev/null +++ b/depends/packages/crate_thiserror.mk @@ -0,0 +1,15 @@ +package=crate_thiserror +$(package)_crate_name=thiserror +$(package)_version=1.0.20 +$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) +$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate +$(package)_sha256_hash=7dfdd070ccd8ccb78f4ad66bf1982dc37f620ef696c6b5028fe2ed83dd3d0d08 +$(package)_crate_versioned_name=$($(package)_crate_name) + +define $(package)_preprocess_cmds + $(call generate_crate_checksum,$(package)) +endef + +define $(package)_stage_cmds + $(call vendor_crate_source,$(package)) +endef diff --git a/depends/packages/crate_thiserror_impl.mk b/depends/packages/crate_thiserror_impl.mk new file mode 100644 index 000000000..4d7f8b475 --- /dev/null +++ b/depends/packages/crate_thiserror_impl.mk @@ -0,0 +1,15 @@ +package=crate_thiserror_impl +$(package)_crate_name=thiserror-impl +$(package)_version=1.0.20 +$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) +$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate +$(package)_sha256_hash=bd80fc12f73063ac132ac92aceea36734f04a1d93c1240c6944e23a3b8841793 +$(package)_crate_versioned_name=$($(package)_crate_name) + +define $(package)_preprocess_cmds + $(call generate_crate_checksum,$(package)) +endef + +define $(package)_stage_cmds + $(call vendor_crate_source,$(package)) +endef diff --git a/depends/packages/crate_zeroize.mk b/depends/packages/crate_zeroize.mk new file mode 100644 index 000000000..ffdb96dde --- /dev/null +++ b/depends/packages/crate_zeroize.mk @@ -0,0 +1,15 @@ +package=crate_zeroize +$(package)_crate_name=zeroize +$(package)_version=1.1.0 +$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) +$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate +$(package)_sha256_hash=3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8 +$(package)_crate_versioned_name=$($(package)_crate_name) + +define $(package)_preprocess_cmds + $(call generate_crate_checksum,$(package)) +endef + +define $(package)_stage_cmds + $(call vendor_crate_source,$(package)) +endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 5826a5133..4679e8886 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -7,6 +7,7 @@ rust_crates := \ crate_autocfg \ crate_bellman \ crate_bigint \ + crate_bincode \ crate_bit_vec \ crate_blake2b_simd \ crate_blake2s_simd \ @@ -27,8 +28,10 @@ rust_crates := \ crate_crunchy \ crate_crypto_api_chachapoly \ crate_crypto_api \ + crate_curve25519_dalek \ crate_digest \ crate_directories \ + crate_ed25519_zebra \ crate_fake_simd \ crate_ff_derive \ crate_ff \ @@ -39,6 +42,7 @@ rust_crates := \ crate_getrandom \ crate_group \ crate_hex \ + crate_hex2 \ crate_lazy_static \ crate_libc \ crate_log \ @@ -62,8 +66,13 @@ rust_crates := \ crate_scopeguard \ crate_semver_parser \ crate_semver \ + crate_serde \ + crate_serde_derive \ crate_sha2 \ + crate_subtle \ crate_syn \ + crate_thiserror \ + crate_thiserror_impl \ crate_typenum \ crate_unicode_xid \ crate_wasi \ @@ -72,7 +81,8 @@ rust_crates := \ crate_winapi_x86_64_pc_windows_gnu \ crate_zcash_history \ crate_zcash_primitives \ - crate_zcash_proofs + crate_zcash_proofs \ + crate_zeroize rust_packages := rust $(rust_crates) proton_packages := proton zcash_packages := libsodium utfcpp diff --git a/src/gtest/test_consensus.cpp b/src/gtest/test_consensus.cpp index 46dd730dd..a8e464487 100644 --- a/src/gtest/test_consensus.cpp +++ b/src/gtest/test_consensus.cpp @@ -1,5 +1,6 @@ #include #include +#include "librustzcash.h" #include "uint256.h" #include "utilstrencodings.h" @@ -18,6 +19,13 @@ void TestLibsodiumEd25519SignatureVerification( (const unsigned char*)msg.data(), msg.size(), pubkey.data()), 0); + + EXPECT_EQ( + librustzcash_zebra_crypto_sign_verify_detached( + sig.data(), + (const unsigned char*)msg.data(), msg.size(), + pubkey.data()), + 0); } TEST(ConsensusTests, LibsodiumPubkeyValidation) { diff --git a/src/rust/include/librustzcash.h b/src/rust/include/librustzcash.h index f69ae6c6d..bc43f8601 100644 --- a/src/rust/include/librustzcash.h +++ b/src/rust/include/librustzcash.h @@ -351,6 +351,13 @@ extern "C" { const unsigned char *n_ptr, unsigned char *h_ret ); + + int librustzcash_zebra_crypto_sign_verify_detached( + const unsigned char *sig, + const unsigned char *m, + unsigned long long mlen, + const unsigned char *pk + ); #ifdef __cplusplus } #endif diff --git a/src/rust/src/rustzcash.rs b/src/rust/src/rustzcash.rs index 8ef52a751..383a69c12 100644 --- a/src/rust/src/rustzcash.rs +++ b/src/rust/src/rustzcash.rs @@ -1340,3 +1340,42 @@ pub extern "system" fn librustzcash_mmr_hash_node( 0 } + +#[no_mangle] +pub extern "system" fn librustzcash_zebra_crypto_sign_verify_detached( + sig: *const [u8; 64], + m: *const u8, + mlen: u64, + pk: *const [u8; 32], +) -> isize { + use ed25519_zebra::{Signature, VerificationKey, VerificationKeyBytes}; + use std::convert::TryFrom; + + let sig = Signature::from(*unsafe { + match sig.as_ref() { + Some(sig) => sig, + None => return 1, + } + }); + + let pk: VerificationKeyBytes = bincode::deserialize(unsafe { + match pk.as_ref() { + Some(pk) => pk, + None => return 1, + } + }) + .expect("should never fail to deserialize raw bytes"); + + let pk = match VerificationKey::try_from(pk) { + Ok(pk) => pk, + Err(_) => return 1, + }; + + let m = unsafe { slice::from_raw_parts(m, mlen as usize) }; + + if pk.verify(&sig, m).is_err() { + 1 + } else { + 0 + } +} From 493c0f98a26abd2b5a215a08fde9cb78d999426e Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Tue, 23 Jun 2020 11:47:29 -0300 Subject: [PATCH 071/725] change order of returned pair, fix compatibility issue Co-Authored-By: Daira Hopwood --- src/init.cpp | 2 +- src/main.h | 2 +- src/rpc/mining.cpp | 6 ++---- src/rpc/misc.cpp | 6 ++---- src/rpc/net.cpp | 6 ++---- src/test/alert_tests.cpp | 22 +++++++++++----------- src/warnings.cpp | 34 ++++++++++++++++------------------ src/warnings.h | 4 ++-- 8 files changed, 37 insertions(+), 45 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 02cc0f9fa..97b665831 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -324,7 +324,7 @@ void OnRPCStopped() void OnRPCPreCommand(const CRPCCommand& cmd) { // Observe safe mode - string strWarning = GetWarnings("rpc").second; + string strWarning = GetWarnings("rpc").first; if (strWarning != "" && !GetBoolArg("-disablesafemode", DEFAULT_DISABLE_SAFEMODE) && !cmd.okSafeMode) throw JSONRPCError(RPC_FORBIDDEN_BY_SAFE_MODE, string("Safe mode: ") + strWarning); diff --git a/src/main.h b/src/main.h index 12704a3a3..1f827c4ea 100644 --- a/src/main.h +++ b/src/main.h @@ -250,7 +250,7 @@ void PartitionCheck(bool (*initialDownloadCheck)(const CChainParams&), CCritical /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload(const CChainParams& chainParams); /** Pair of timestamp and formatted string that describes several potential problems detected by the core */ -std::pair GetWarnings(const std::string& strFor); +std::pair GetWarnings(const std::string& strFor); /** Retrieve a transaction (from memory pool, or from disk, if possible) */ bool GetTransaction(const uint256 &hash, CTransaction &tx, const Consensus::Params& params, uint256 &hashBlock, bool fAllowSlow = false); /** Find the best known block, and make it the tip of the block chain */ diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index b0e5d0476..d15b2079e 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -341,11 +341,9 @@ UniValue getmininginfo(const UniValue& params, bool fHelp) obj.pushKV("currentblocksize", (uint64_t)nLastBlockSize); obj.pushKV("currentblocktx", (uint64_t)nLastBlockTx); obj.pushKV("difficulty", (double)GetNetworkDifficulty()); - UniValue wobj(UniValue::VOBJ); auto warnings = GetWarnings("statusbar"); - wobj.pushKV("timestamp", warnings.first); - wobj.pushKV("msg", warnings.second); - obj.pushKV("errors", wobj); + obj.pushKV("errors", warnings.first); + obj.pushKV("errorstimestamp", warnings.second); obj.pushKV("genproclimit", (int)GetArg("-genproclimit", DEFAULT_GENERATE_THREADS)); obj.pushKV("localsolps" , getlocalsolps(params, false)); obj.pushKV("networksolps", getnetworksolps(params, false)); diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 6268936e0..6de324754 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -105,11 +105,9 @@ UniValue getinfo(const UniValue& params, bool fHelp) obj.pushKV("paytxfee", ValueFromAmount(payTxFee.GetFeePerK())); #endif obj.pushKV("relayfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); - UniValue wobj(UniValue::VOBJ); auto warnings = GetWarnings("statusbar"); - wobj.pushKV("timestamp", warnings.first); - wobj.pushKV("msg", warnings.second); - obj.pushKV("errors", wobj); + obj.pushKV("errors", warnings.first); + obj.pushKV("errorstimestamp", warnings.second); return obj; } diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index a66eee589..ef36c0f3a 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -488,11 +488,9 @@ UniValue getnetworkinfo(const UniValue& params, bool fHelp) } } obj.pushKV("localaddresses", localAddresses); - UniValue wobj(UniValue::VOBJ); auto warnings = GetWarnings("statusbar"); - wobj.pushKV("timestamp", warnings.first); - wobj.pushKV("msg", warnings.second); - obj.pushKV("errors", wobj); + obj.pushKV("errors", warnings.first); + obj.pushKV("errorstimestamp",warnings.second); return obj; } diff --git a/src/test/alert_tests.cpp b/src/test/alert_tests.cpp index da1c47885..f0c6c7e4d 100644 --- a/src/test/alert_tests.cpp +++ b/src/test/alert_tests.cpp @@ -395,17 +395,17 @@ BOOST_AUTO_TEST_CASE(AlertDisablesRPC) const std::vector& alertKey = Params(CBaseChainParams::MAIN).AlertKey(); // Command should work before alerts - BOOST_CHECK_EQUAL(GetWarnings("rpc").second, ""); + BOOST_CHECK_EQUAL(GetWarnings("rpc").first, ""); // First alert should disable RPC alerts[7].ProcessAlert(alertKey, false); BOOST_CHECK_EQUAL(alerts[7].strRPCError, "RPC disabled"); - BOOST_CHECK_EQUAL(GetWarnings("rpc").second, "RPC disabled"); + BOOST_CHECK_EQUAL(GetWarnings("rpc").first, "RPC disabled"); // Second alert should re-enable RPC alerts[8].ProcessAlert(alertKey, false); BOOST_CHECK_EQUAL(alerts[8].strRPCError, ""); - BOOST_CHECK_EQUAL(GetWarnings("rpc").second, ""); + BOOST_CHECK_EQUAL(GetWarnings("rpc").first, ""); SetMockTime(0); mapAlerts.clear(); @@ -438,7 +438,7 @@ void PartitionAlertTestImpl(const Consensus::Params& params, int startTime, int // as normal, no worries: SetMiscWarning("", 0); PartitionCheck(falseFunc, csDummy, &indexDummy[799]); - BOOST_CHECK_EQUAL("", GetMiscWarning().second); + BOOST_CHECK_EQUAL("", GetMiscWarning().first); // Test 2: go 3.5 hours without a block, expect a warning: now += 3*60*60+30*60; @@ -449,9 +449,9 @@ void PartitionAlertTestImpl(const Consensus::Params& params, int startTime, int auto warning = GetMiscWarning(); // advance 5 seconds so alert time will be in the past SetMockTime(now + 5); - BOOST_CHECK_EQUAL(GetTime() - 5, warning.first); - BOOST_CHECK_EQUAL(expectedSlowErr, warning.second); - BOOST_TEST_MESSAGE(DateTimeStrFormat("%Y-%m-%d %H:%M:%S", warning.first) + " - Got alert text: " + warning.second); + BOOST_CHECK_EQUAL(GetTime() - 5, warning.second); + BOOST_CHECK_EQUAL(expectedSlowErr, warning.first); + BOOST_TEST_MESSAGE(DateTimeStrFormat("%Y-%m-%d %H:%M:%S", warning.second) + " - Got alert text: " + warning.first); // Test 3: test the "partition alerts only go off once per day" // code: @@ -459,7 +459,7 @@ void PartitionAlertTestImpl(const Consensus::Params& params, int startTime, int SetMockTime(now); SetMiscWarning("", 0); PartitionCheck(falseFunc, csDummy, &indexDummy[799]); - BOOST_CHECK_EQUAL("", GetMiscWarning().second); + BOOST_CHECK_EQUAL("", GetMiscWarning().first); // Test 4: get 2.5 times as many blocks as expected: start = now + 60*60*24; // Pretend it is a day later @@ -476,9 +476,9 @@ void PartitionAlertTestImpl(const Consensus::Params& params, int startTime, int warning = GetMiscWarning(); // advance 5 seconds so alert time will be in the past SetMockTime(now + 5); - BOOST_CHECK_EQUAL(GetTime() - 5, warning.first); - BOOST_CHECK_EQUAL(expectedFastErr, warning.second); - BOOST_TEST_MESSAGE(DateTimeStrFormat("%Y-%m-%d %H:%M:%S", warning.first) + " - Got alert text: " + warning.second); + BOOST_CHECK_EQUAL(GetTime() - 5, warning.second); + BOOST_CHECK_EQUAL(expectedFastErr, warning.first); + BOOST_TEST_MESSAGE(DateTimeStrFormat("%Y-%m-%d %H:%M:%S", warning.second) + " - Got alert text: " + warning.first); SetMiscWarning("", 0); SetMockTime(0); diff --git a/src/warnings.cpp b/src/warnings.cpp index 9a52e488c..e80c067fb 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -23,12 +23,10 @@ void SetMiscWarning(const std::string& strWarning, int64_t timestamp) timestampWarning = timestamp; } -std::pair GetMiscWarning() +std::pair GetMiscWarning() { LOCK(cs_warnings); - std::pair misc; - misc.first = timestampWarning; - misc.second = strMiscWarning; + std::pair misc(strMiscWarning, timestampWarning); return misc; } @@ -56,39 +54,39 @@ bool GetfLargeWorkInvalidChainFound() return fLargeWorkInvalidChainFound; } -std::pair GetWarnings(const std::string& strFor) +std::pair GetWarnings(const std::string& strFor) { - std::pair rpc; - std::pair statusbar; - rpc.first = GetTime(); - statusbar.first = GetTime(); + std::pair rpc; + std::pair statusbar; + rpc.second = GetTime(); + statusbar.second = GetTime(); int nPriority = 0; LOCK(cs_warnings); if (!CLIENT_VERSION_IS_RELEASE) - statusbar.second = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); + statusbar.first = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); if (GetBoolArg("-testsafemode", DEFAULT_TESTSAFEMODE)) - statusbar.second = rpc.second = "testsafemode enabled"; + statusbar.first = rpc.first = "testsafemode enabled"; // Misc warnings like out of disk space and clock is wrong if (strMiscWarning != "") { nPriority = 1000; - statusbar.first = timestampWarning; - statusbar.second = strMiscWarning; + statusbar.first = strMiscWarning; + statusbar.second = timestampWarning; } if (fLargeWorkForkFound) { nPriority = 2000; - statusbar.second = rpc.second = _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."); + statusbar.first = rpc.first = _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."); } else if (fLargeWorkInvalidChainFound) { nPriority = 2000; - statusbar.second = rpc.second = _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."); + statusbar.first = rpc.first = _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."); } // Alerts @@ -100,9 +98,9 @@ std::pair GetWarnings(const std::string& strFor) if (alert.AppliesToMe() && alert.nPriority > nPriority) { nPriority = alert.nPriority; - statusbar.second = alert.strStatusBar; + statusbar.first = alert.strStatusBar; if (alert.nPriority >= ALERT_PRIORITY_SAFE_MODE) { - rpc.second = alert.strRPCError; + rpc.first = alert.strRPCError; } } } @@ -113,5 +111,5 @@ std::pair GetWarnings(const std::string& strFor) else if (strFor == "rpc") return rpc; assert(!"GetWarnings(): invalid parameter"); - return std::make_pair(0, "error"); + return std::make_pair("error", GetTime()); } diff --git a/src/warnings.h b/src/warnings.h index ddbe3cf41..db44aabde 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -10,12 +10,12 @@ #include void SetMiscWarning(const std::string& strWarning, int64_t timestamp); -std::pair GetMiscWarning(); +std::pair GetMiscWarning(); void SetfLargeWorkForkFound(bool flag); bool GetfLargeWorkForkFound(); void SetfLargeWorkInvalidChainFound(bool flag); bool GetfLargeWorkInvalidChainFound(); -std::pair GetWarnings(const std::string& strFor); +std::pair GetWarnings(const std::string& strFor); static const bool DEFAULT_TESTSAFEMODE = false; From 76c416d2df3190e9fa29aec77646db0c110de003 Mon Sep 17 00:00:00 2001 From: Daira Hopwood Date: Tue, 23 Jun 2020 19:38:29 +0100 Subject: [PATCH 072/725] Remove an unused CCriticalSection. The code that used this was removed in 5f84491d829e26a9c676a4e7acfff1feb37f5545. Signed-off-by: Daira Hopwood --- src/zcash/JoinSplit.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/zcash/JoinSplit.cpp b/src/zcash/JoinSplit.cpp index 0a052209a..2382a28a8 100644 --- a/src/zcash/JoinSplit.cpp +++ b/src/zcash/JoinSplit.cpp @@ -20,8 +20,6 @@ namespace libzcash { -static CCriticalSection cs_ParamsIO; - template class JoinSplitCircuit : public JoinSplit { public: From 9f71fa1c09692ae985ac49f29770fee62d0b0b99 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Tue, 23 Jun 2020 12:24:45 -0600 Subject: [PATCH 073/725] Change to version of ed25519-zebra crate which is compliant with ZIP 215. --- Cargo.lock | 3 +-- Cargo.toml | 2 +- depends/packages/crate_ed25519_zebra.mk | 13 +++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 651752348..83d226fbc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -306,8 +306,7 @@ dependencies = [ [[package]] name = "ed25519-zebra" version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76f15c88332faad36abb368aca89deb5cc4f440e5181c8848f8bdd049848f7b" +source = "git+https://github.com/ebfull/ed25519-zebra?rev=8c97acde89b6446f7f2505069fc7b9ac3a541417#8c97acde89b6446f7f2505069fc7b9ac3a541417" dependencies = [ "curve25519-dalek", "hex 0.4.2", diff --git a/Cargo.toml b/Cargo.toml index 3406b28b1..aeadaa039 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ rand_core = "0.5.1" zcash_history = "0.2" zcash_primitives = "0.2" zcash_proofs = "0.2" -ed25519-zebra = "0.4.1" +ed25519-zebra = { git = "https://github.com/ebfull/ed25519-zebra", rev = "8c97acde89b6446f7f2505069fc7b9ac3a541417" } bincode = "1.2.1" [profile.release] diff --git a/depends/packages/crate_ed25519_zebra.mk b/depends/packages/crate_ed25519_zebra.mk index f1c12a2db..96938e3bc 100644 --- a/depends/packages/crate_ed25519_zebra.mk +++ b/depends/packages/crate_ed25519_zebra.mk @@ -1,15 +1,16 @@ package=crate_ed25519_zebra $(package)_crate_name=ed25519-zebra -$(package)_version=0.4.1 -$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) -$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate -$(package)_sha256_hash=a76f15c88332faad36abb368aca89deb5cc4f440e5181c8848f8bdd049848f7b +$(package)_download_path=https://github.com/ebfull/$($(package)_crate_name)/archive/ +$(package)_file_name=$(package)-$($(package)_git_commit).tar.gz +$(package)_download_file=$($(package)_git_commit).tar.gz +$(package)_sha256_hash=e0b432e0161abe9cb27cf96dd42c1486d32297e2fdbf725b70e7ae1f5b53d052 +$(package)_git_commit=8c97acde89b6446f7f2505069fc7b9ac3a541417 $(package)_crate_versioned_name=$($(package)_crate_name) define $(package)_preprocess_cmds - $(call generate_crate_checksum,$(package)) + $(call generate_unpackaged_crate_checksum,$(package)) endef define $(package)_stage_cmds $(call vendor_crate_source,$(package)) -endef +endef \ No newline at end of file From 3251bea45a661dfd3a6918b76d7e413026216b71 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Tue, 23 Jun 2020 12:41:28 -0600 Subject: [PATCH 074/725] Enforce ZIP 215 rules upon activation of Canopy. --- src/main.cpp | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index d11ca5bae..781430328 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -792,6 +792,7 @@ bool ContextualCheckTransaction( bool saplingActive = chainparams.GetConsensus().NetworkUpgradeActive(nHeight, Consensus::UPGRADE_SAPLING); bool isSprout = !overwinterActive; bool heartwoodActive = chainparams.GetConsensus().NetworkUpgradeActive(nHeight, Consensus::UPGRADE_HEARTWOOD); + bool canopyActive = chainparams.GetConsensus().NetworkUpgradeActive(nHeight, Consensus::UPGRADE_CANOPY); // If Sprout rules apply, reject transactions which are intended for Overwinter and beyond if (isSprout && tx.fOverwintered) { @@ -958,25 +959,39 @@ bool ContextualCheckTransaction( } } + int (*ed25519_verifier)( + const unsigned char *, + const unsigned char *, + unsigned long long , + const unsigned char * + ) = &crypto_sign_verify_detached; + + // Switch from using the libsodium ed25519 verifier to using the + // ed25519-zebra Rust crate, which implements an ed25519 verifier that is + // compliant with ZIP 215. + if (canopyActive) { + ed25519_verifier = &librustzcash_zebra_crypto_sign_verify_detached; + } + if (!tx.vJoinSplit.empty()) { BOOST_STATIC_ASSERT(crypto_sign_PUBLICKEYBYTES == 32); // We rely on libsodium to check that the signature is canonical. // https://github.com/jedisct1/libsodium/commit/62911edb7ff2275cccd74bf1c8aefcc4d76924e0 - if (crypto_sign_verify_detached(&tx.joinSplitSig[0], - dataToBeSigned.begin(), 32, - tx.joinSplitPubKey.begin() - ) != 0) { + if (ed25519_verifier(&tx.joinSplitSig[0], + dataToBeSigned.begin(), 32, + tx.joinSplitPubKey.begin() + ) != 0) { // Check whether the failure was caused by an outdated consensus // branch ID; if so, inform the node that they need to upgrade. We // only check the previous epoch's branch ID, on the assumption that // users creating transactions will notice their transactions // failing before a second network upgrade occurs. - if (crypto_sign_verify_detached(&tx.joinSplitSig[0], - prevDataToBeSigned.begin(), 32, - tx.joinSplitPubKey.begin() - ) == 0) { + if (ed25519_verifier(&tx.joinSplitSig[0], + prevDataToBeSigned.begin(), 32, + tx.joinSplitPubKey.begin() + ) == 0) { return state.DoS( dosLevelPotentiallyRelaxing, false, REJECT_INVALID, strprintf( "old-consensus-branch-id (Expected %s, found %s)", From 2a3527bfe82ff469c639fab84075c488443c5e59 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Tue, 23 Jun 2020 14:31:41 -0600 Subject: [PATCH 075/725] Add test that a weird signature successfully validates. --- Cargo.lock | 1 + Cargo.toml | 3 +++ src/rust/src/tests/ed25519.rs | 46 +++++++++++++++++++++++++++++++++++ src/rust/src/tests/mod.rs | 1 + 4 files changed, 51 insertions(+) create mode 100644 src/rust/src/tests/ed25519.rs diff --git a/Cargo.lock b/Cargo.lock index 83d226fbc..ff2752cba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -439,6 +439,7 @@ dependencies = [ "bincode", "blake2b_simd", "blake2s_simd", + "curve25519-dalek", "ed25519-zebra", "ff", "lazy_static", diff --git a/Cargo.toml b/Cargo.toml index aeadaa039..258b09ef1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,9 @@ zcash_proofs = "0.2" ed25519-zebra = { git = "https://github.com/ebfull/ed25519-zebra", rev = "8c97acde89b6446f7f2505069fc7b9ac3a541417" } bincode = "1.2.1" +[dev-dependencies] +curve25519-dalek = "2.1.0" + [profile.release] lto = true panic = 'abort' diff --git a/src/rust/src/tests/ed25519.rs b/src/rust/src/tests/ed25519.rs new file mode 100644 index 000000000..0c1d7f13b --- /dev/null +++ b/src/rust/src/tests/ed25519.rs @@ -0,0 +1,46 @@ +use ed25519_zebra::{Signature, VerificationKey, VerificationKeyBytes}; +use curve25519_dalek::{scalar::Scalar, edwards::{CompressedEdwardsY}}; +use crate::librustzcash_zebra_crypto_sign_verify_detached; + +#[test] +fn test_weird_signature() { + // This is a signature from hell. + + // A = (0, 1) encoded with high bit set, even though the x-coordinate is 0 + let pk = [ + 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80 + ]; + + // R = (0, -1) encoded with high bit set, even though the x-coordinate is 0 + // s = 0 + let sig = [ + 0xec, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ]; + + // [8]R = [8]([s]B - [k]A) + assert_eq!(librustzcash_zebra_crypto_sign_verify_detached( + &sig, &0xff, 1, &pk + ), 0); +} diff --git a/src/rust/src/tests/mod.rs b/src/rust/src/tests/mod.rs index bbaee0abb..5c869d008 100644 --- a/src/rust/src/tests/mod.rs +++ b/src/rust/src/tests/mod.rs @@ -7,6 +7,7 @@ mod key_components; mod mmr; mod notes; mod signatures; +mod ed25519; #[test] fn sapling_generators() { From 14bdf7e8118c60cc05886de2479dc9ac2a4b1de0 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Tue, 23 Jun 2020 15:24:51 -0600 Subject: [PATCH 076/725] Remove bincode crate. --- Cargo.lock | 11 ----------- Cargo.toml | 1 - depends/packages/crate_bincode.mk | 15 --------------- depends/packages/packages.mk | 1 - src/rust/src/rustzcash.rs | 13 ++++--------- 5 files changed, 4 insertions(+), 37 deletions(-) delete mode 100644 depends/packages/crate_bincode.mk diff --git a/Cargo.lock b/Cargo.lock index ff2752cba..3f1cb5de6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,16 +82,6 @@ dependencies = [ "crunchy", ] -[[package]] -name = "bincode" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" -dependencies = [ - "byteorder", - "serde", -] - [[package]] name = "bit-vec" version = "0.4.4" @@ -436,7 +426,6 @@ name = "librustzcash" version = "0.2.0" dependencies = [ "bellman", - "bincode", "blake2b_simd", "blake2s_simd", "curve25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 258b09ef1..ce05caafd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,6 @@ zcash_history = "0.2" zcash_primitives = "0.2" zcash_proofs = "0.2" ed25519-zebra = { git = "https://github.com/ebfull/ed25519-zebra", rev = "8c97acde89b6446f7f2505069fc7b9ac3a541417" } -bincode = "1.2.1" [dev-dependencies] curve25519-dalek = "2.1.0" diff --git a/depends/packages/crate_bincode.mk b/depends/packages/crate_bincode.mk deleted file mode 100644 index fa8d8cdb6..000000000 --- a/depends/packages/crate_bincode.mk +++ /dev/null @@ -1,15 +0,0 @@ -package=crate_bincode -$(package)_crate_name=bincode -$(package)_version=1.2.1 -$(package)_download_path=https://static.crates.io/crates/$($(package)_crate_name) -$(package)_file_name=$($(package)_crate_name)-$($(package)_version).crate -$(package)_sha256_hash=5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf -$(package)_crate_versioned_name=$($(package)_crate_name) - -define $(package)_preprocess_cmds - $(call generate_crate_checksum,$(package)) -endef - -define $(package)_stage_cmds - $(call vendor_crate_source,$(package)) -endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 4679e8886..75bcb4de0 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -7,7 +7,6 @@ rust_crates := \ crate_autocfg \ crate_bellman \ crate_bigint \ - crate_bincode \ crate_bit_vec \ crate_blake2b_simd \ crate_blake2s_simd \ diff --git a/src/rust/src/rustzcash.rs b/src/rust/src/rustzcash.rs index 383a69c12..7ac5e7a4e 100644 --- a/src/rust/src/rustzcash.rs +++ b/src/rust/src/rustzcash.rs @@ -1358,15 +1358,10 @@ pub extern "system" fn librustzcash_zebra_crypto_sign_verify_detached( } }); - let pk: VerificationKeyBytes = bincode::deserialize(unsafe { - match pk.as_ref() { - Some(pk) => pk, - None => return 1, - } - }) - .expect("should never fail to deserialize raw bytes"); - - let pk = match VerificationKey::try_from(pk) { + let pk = match VerificationKey::try_from(*match unsafe { pk.as_ref() } { + Some(pk) => pk, + None => return 1, + }) { Ok(pk) => pk, Err(_) => return 1, }; From ba044509455621bc5b833fe0a289242480f04950 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Tue, 23 Jun 2020 16:29:51 -0600 Subject: [PATCH 077/725] Remove unused curve25519-dalek dev-dependency. --- Cargo.lock | 1 - Cargo.toml | 3 --- src/rust/src/tests/ed25519.rs | 1 - 3 files changed, 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f1cb5de6..e62b446e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -428,7 +428,6 @@ dependencies = [ "bellman", "blake2b_simd", "blake2s_simd", - "curve25519-dalek", "ed25519-zebra", "ff", "lazy_static", diff --git a/Cargo.toml b/Cargo.toml index ce05caafd..b1eb20184 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,9 +33,6 @@ zcash_primitives = "0.2" zcash_proofs = "0.2" ed25519-zebra = { git = "https://github.com/ebfull/ed25519-zebra", rev = "8c97acde89b6446f7f2505069fc7b9ac3a541417" } -[dev-dependencies] -curve25519-dalek = "2.1.0" - [profile.release] lto = true panic = 'abort' diff --git a/src/rust/src/tests/ed25519.rs b/src/rust/src/tests/ed25519.rs index 0c1d7f13b..35511900b 100644 --- a/src/rust/src/tests/ed25519.rs +++ b/src/rust/src/tests/ed25519.rs @@ -1,5 +1,4 @@ use ed25519_zebra::{Signature, VerificationKey, VerificationKeyBytes}; -use curve25519_dalek::{scalar::Scalar, edwards::{CompressedEdwardsY}}; use crate::librustzcash_zebra_crypto_sign_verify_detached; #[test] From f29c731915673efa9c2226b6877d8b9f50ebfd62 Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Tue, 23 Jun 2020 16:32:56 -0600 Subject: [PATCH 078/725] Minor adjustments to librustzcash and tests. --- src/rust/src/rustzcash.rs | 2 +- src/rust/src/tests/ed25519.rs | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/rust/src/rustzcash.rs b/src/rust/src/rustzcash.rs index 7ac5e7a4e..c80649fca 100644 --- a/src/rust/src/rustzcash.rs +++ b/src/rust/src/rustzcash.rs @@ -1348,7 +1348,7 @@ pub extern "system" fn librustzcash_zebra_crypto_sign_verify_detached( mlen: u64, pk: *const [u8; 32], ) -> isize { - use ed25519_zebra::{Signature, VerificationKey, VerificationKeyBytes}; + use ed25519_zebra::{Signature, VerificationKey}; use std::convert::TryFrom; let sig = Signature::from(*unsafe { diff --git a/src/rust/src/tests/ed25519.rs b/src/rust/src/tests/ed25519.rs index 35511900b..53d1f04ab 100644 --- a/src/rust/src/tests/ed25519.rs +++ b/src/rust/src/tests/ed25519.rs @@ -1,4 +1,3 @@ -use ed25519_zebra::{Signature, VerificationKey, VerificationKeyBytes}; use crate::librustzcash_zebra_crypto_sign_verify_detached; #[test] @@ -19,7 +18,7 @@ fn test_weird_signature() { // R = (0, -1) encoded with high bit set, even though the x-coordinate is 0 // s = 0 - let sig = [ + let mut sig = [ 0xec, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, @@ -38,8 +37,14 @@ fn test_weird_signature() { 0x00, 0x00, 0x00, 0x00 ]; - // [8]R = [8]([s]B - [k]A) assert_eq!(librustzcash_zebra_crypto_sign_verify_detached( &sig, &0xff, 1, &pk ), 0); + + // Screw with the signature to exercise verification failure + sig[32] = 0x01; // wrong (but a valid) s value + + assert_eq!(librustzcash_zebra_crypto_sign_verify_detached( + &sig, &0xff, 1, &pk + ), 1); } From 26522d212b4b84ae4cbbdde8c7ab0ff55577a591 Mon Sep 17 00:00:00 2001 From: Alfredo Garcia Date: Wed, 24 Jun 2020 12:21:25 -0300 Subject: [PATCH 079/725] only allow duplicates for certain options of the config --- src/util.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/util.cpp b/src/util.cpp index 82d705ea8..5d88aba88 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -627,10 +627,21 @@ void ReadConfigFile(const std::string& confPath, set setOptions; setOptions.insert("*"); + const vector allowed_duplicates = {"addnode", "connect", "rpcallowip"}; + set unique_options; + for (boost::program_options::detail::config_file_iterator it(streamConfig, setOptions), end; it != end; ++it) { string strKey = string("-") + it->string_key; string strValue = it->value[0]; + + if (find(allowed_duplicates.begin(), allowed_duplicates.end(), it->string_key) == allowed_duplicates.end()) + { + if (!unique_options.insert(strKey).second) { + throw std::runtime_error(strprintf("Not allowed duplicated option %s found.", strKey)); + } + } + InterpretNegativeSetting(strKey, strValue); // Don't overwrite existing settings so command line settings override zcash.conf if (mapSettingsRet.count(strKey) == 0) From 633594478481acde6df27c664a9f9fc95b0f4595 Mon Sep 17 00:00:00 2001 From: Larry Ruane Date: Wed, 24 Jun 2020 09:27:11 -0600 Subject: [PATCH 080/725] add python test to reproduce bug 4301 --- qa/pull-tester/rpc-tests.sh | 1 + qa/rpc-tests/wallet_db_flush.py | 87 +++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100755 qa/rpc-tests/wallet_db_flush.py diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index b6454a304..9c3666f5a 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -105,6 +105,7 @@ testScriptsExt=( 'invalidblockrequest.py' # 'forknotify.py' 'p2p-acceptblock.py' + 'wallet_db_flush.py' ); if [ "x$ENABLE_ZMQ" = "x1" ]; then diff --git a/qa/rpc-tests/wallet_db_flush.py b/qa/rpc-tests/wallet_db_flush.py new file mode 100755 index 000000000..d66e4ffb7 --- /dev/null +++ b/qa/rpc-tests/wallet_db_flush.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Zcash developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://www.opensource.org/licenses/mit-license.php . +# +# This test reproduces https://github.com/zcash/zcash/issues/4301 +# It takes an hour to run! + +from decimal import Decimal +from test_framework.authproxy import JSONRPCException +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + bitcoind_processes, + initialize_chain_clean, + start_node, +) +import time + +class WalletDBFlush (BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 1) + + def start_node_with(self, index, extra_args=[]): + args = [ + "-nuparams=2bb40e60:1", # Blossom + "-nuparams=f5b9230b:2", # Heartwood + "-nurejectoldversions=false", + ] + return start_node(index, self.options.tmpdir, args + extra_args) + + def setup_network(self, split=False): + self.nodes = [] + self.nodes.append(self.start_node_with(0)) + self.is_network_split=False + self.sync_all() + + def run_test (self): + print("PLEASE NOTE: This test takes an hour to run!") + + # This test requires shielded funds in the local wallet so + # there is witness data, and the easiest way to get shielded + # funds is to mine (since Heartwood, mining reward can go to + # a zaddr), so first create a Sapling address to mine to. + zaddr = self.nodes[0].z_getnewaddress('sapling') + self.nodes[0].generate(2) + + self.nodes[0].stop() + bitcoind_processes[0].wait() + + print("Start mining to address ", zaddr) + self.nodes[0] = self.start_node_with(0, [ + "-mineraddress=%s" % zaddr, + ]) + self.nodes[0].generate(1) + time.sleep(4) + self.nodes[0].stop() + bitcoind_processes[0].wait() + + # If you replace main.cpp:3129 DATABASE_WRITE_INTERVAL with + # 60 (seconds), then sleeptime here can be 80, and this test + # will fail (pre-PR) much faster. + sleeptime = 3620 # just over one hour (DATABASE_WRITE_INTERVAL) + + print("Restart, sleep {}, mine (pre-PR will flush bad wallet state)".format(sleeptime)) + self.nodes[0] = self.start_node_with(0, [ + "-mineraddress=%s" % zaddr, + ]) + assert_equal(self.nodes[0].z_getbalance(zaddr, 0), 5) + time.sleep(sleeptime) + self.nodes[0].generate(1) + time.sleep(4) + self.nodes[0].stop() + bitcoind_processes[0].wait() + + print("Restart, generate, expect assert in CopyPreviousWitnesses") + self.nodes[0] = self.start_node_with(0, [ + "-mineraddress=%s" % zaddr, + ]) + self.nodes[0].generate(1) + time.sleep(4) + self.nodes[0].stop() + +if __name__ == '__main__': + WalletDBFlush().main() From 98c6bdbae841ffb6fefce8b9156b8fea24b98eee Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Wed, 24 Jun 2020 13:55:21 -0600 Subject: [PATCH 081/725] Redirect git checkouts of ebfull/ed25519-zebra through our vendored sources in offline mode. --- .cargo/config.offline | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.cargo/config.offline b/.cargo/config.offline index a6339df37..4cf297a1b 100644 --- a/.cargo/config.offline +++ b/.cargo/config.offline @@ -1,5 +1,10 @@ [source.crates-io] replace-with = "vendored-sources" +[source."https://github.com/ebfull/ed25519-zebra"] +git = "https://github.com/ebfull/ed25519-zebra" +rev = "8c97acde89b6446f7f2505069fc7b9ac3a541417" +replace-with = "vendored-sources" + [source.vendored-sources] # The directory for this source is set to RUST_VENDORED_SOURCES by src/Makefile.am From 0219a971392f58ab7e30a725aec2230393a2125c Mon Sep 17 00:00:00 2001 From: Larry Ruane Date: Wed, 24 Jun 2020 16:49:59 -0600 Subject: [PATCH 082/725] flush witness cache correctly Rather than flushing the witness cache from FlushStateToDisk(), called by ActivateBestChain() called by ProcessNewBlock(), do so from ThreadNotifyWallets() after the wallet has updated the in-memory witness data according to the new block, so it's always consistent on disk. --- src/init.cpp | 3 +++ src/main.cpp | 10 +--------- src/wallet/wallet.cpp | 15 ++++++++++++++- src/wallet/wallet.h | 2 ++ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 1671dba90..631ab3836 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -229,6 +229,9 @@ void Shutdown() if (pcoinsTip != NULL) { FlushStateToDisk(); } + // Flush the wallet witness cache to disk (no longer done by FlushStateToDisk()) + GetMainSignals().SetBestChain(chainActive.GetLocator()); + delete pcoinsTip; pcoinsTip = NULL; delete pcoinscatcher; diff --git a/src/main.cpp b/src/main.cpp index d11ca5bae..cb3b6c7bc 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -3045,7 +3045,6 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { LOCK2(cs_main, cs_LastBlockFile); static int64_t nLastWrite = 0; static int64_t nLastFlush = 0; - static int64_t nLastSetChain = 0; std::set setFilesToPrune; bool fFlushForPrune = false; try { @@ -3068,9 +3067,6 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { if (nLastFlush == 0) { nLastFlush = nNow; } - if (nLastSetChain == 0) { - nLastSetChain = nNow; - } size_t cacheSize = pcoinsTip->DynamicMemoryUsage(); // The cache is large and close to the limit, but we have time now (not in the middle of a block processing). bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize * (10.0/9) > nCoinCacheUsage; @@ -3126,11 +3122,7 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { return AbortNode(state, "Failed to write to coin database"); nLastFlush = nNow; } - if ((mode == FLUSH_STATE_ALWAYS || mode == FLUSH_STATE_PERIODIC) && nNow > nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000) { - // Update best block in wallet (so we can detect restored wallets). - GetMainSignals().SetBestChain(chainActive.GetLocator()); - nLastSetChain = nNow; - } + // Don't flush the wallet witness cache (SetBestChain()) here, see #4301 } catch (const std::runtime_error& e) { return AbortNode(state, std::string("System error while flushing: ") + e.what()); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index a1c7bb5c0..5f6ea468a 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -585,6 +585,19 @@ void CWallet::ChainTipAdded(const CBlockIndex *pindex, { IncrementNoteWitnesses(pindex, pblock, sproutTree, saplingTree); UpdateSaplingNullifierNoteMapForBlock(pblock); + + // SetBestChain() can be expensive for large wallets, so do this + // at most once per hour; the wallet state will be brought up to + // date during rescanning on startup. + int64_t nNow = GetTimeMicros(); + if (nLastSetChain == 0) { + // Don't flush during startup. + nLastSetChain = nNow; + } + if (nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000 < nNow) { + nLastSetChain = nNow; + SetBestChain(chainActive.GetLocator()); + } } void CWallet::ChainTip(const CBlockIndex *pindex, @@ -4641,7 +4654,7 @@ bool CWallet::InitLoadWallet(bool clearWitnessCaches) RegisterValidationInterface(walletInstance); - CBlockIndex *pindexRescan = chainActive.Tip(); + CBlockIndex *pindexRescan; if (clearWitnessCaches || GetBoolArg("-rescan", false)) { walletInstance->ClearNoteWitnessCache(); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c673dfcea..89737ccaa 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -760,6 +760,7 @@ private: int64_t nNextResend; int64_t nLastResend; + int64_t nLastSetChain; bool fBroadcastTransactions; template @@ -920,6 +921,7 @@ public: nOrderPosNext = 0; nNextResend = 0; nLastResend = 0; + nLastSetChain = 0; nTimeFirstKey = 0; fBroadcastTransactions = false; nWitnessCacheSize = 0; From 75876f477b9fd4a24594ad3fe54a76c0af49019d Mon Sep 17 00:00:00 2001 From: Taylor Hornby Date: Wed, 24 Jun 2020 12:20:18 -0600 Subject: [PATCH 083/725] Implement system for postponing dependency updates. --- qa/zcash/postponed-updates.txt | 223 +++++++++++++++++++++++++++++++++ qa/zcash/updatecheck.py | 58 ++++++++- 2 files changed, 276 insertions(+), 5 deletions(-) create mode 100644 qa/zcash/postponed-updates.txt diff --git a/qa/zcash/postponed-updates.txt b/qa/zcash/postponed-updates.txt new file mode 100644 index 000000000..5c1fc6212 --- /dev/null +++ b/qa/zcash/postponed-updates.txt @@ -0,0 +1,223 @@ +# List of Postponed Dependency Updates +# +# The format is: name version expiration, e.g. +# bdb 18.1.40 2020-09-01 +# + +bdb 18.1.40 2020-09-01 +boost 1.71.0 2020-09-01 +boost 1.72.0 2020-09-01 +boost 1.73.0 2020-09-01 +googletest 1.8.1 2020-09-01 +googletest 1.10.0 2020-09-01 +libevent 2.1.10 2020-09-01 +libevent 2.1.11 2020-09-01 +native_ccache 3.3.2 2020-09-01 +native_ccache 3.3.3 2020-09-01 +native_ccache 3.3.4 2020-09-01 +native_ccache 3.3.5 2020-09-01 +native_ccache 3.3.6 2020-09-01 +native_ccache 3.4 2020-09-01 +native_ccache 3.4.1 2020-09-01 +native_ccache 3.4.2 2020-09-01 +native_ccache 3.4.3 2020-09-01 +native_ccache 3.5 2020-09-01 +native_ccache 3.5.1 2020-09-01 +native_ccache 3.6 2020-09-01 +native_ccache 3.7 2020-09-01 +native_ccache 3.7.1 2020-09-01 +native_ccache 3.7.2 2020-09-01 +native_ccache 3.7.3 2020-09-01 +native_ccache 3.7.4 2020-09-01 +native_ccache 3.7.5 2020-09-01 +native_ccache 3.7.6 2020-09-01 +native_ccache 3.7.7 2020-09-01 +native_ccache 3.7.8 2020-09-01 +native_ccache 3.7.9 2020-09-01 +native_ccache 3.7.10 2020-09-01 +openssl 1.1.1.b 2020-09-01 +openssl 1.1.1.c 2020-09-01 +openssl 1.1.1.d 2020-09-01 +openssl 1.1.1.e 2020-09-01 +openssl 1.1.1.f 2020-09-01 +openssl 1.1.1.g 2020-09-01 +proton 0.31.0 2020-09-01 +rust 1.43.0 2020-09-01 +rust 1.43.1 2020-09-01 +rust 1.44.0 2020-09-01 +rust 1.44.1 2020-09-01 +zeromq 4.3.2 2020-09-01 +leveldb 1.19 2020-09-01 +leveldb 1.20 2020-09-01 +utfcpp 3.1.1 2020-09-01 +crate_aes 0.4.0 2020-09-01 +crate_aesni 0.7.0 2020-09-01 +crate_aes_soft 0.4.0 2020-09-01 +crate_arrayvec 0.4.12 2020-09-01 +crate_arrayvec 0.5.0 2020-09-01 +crate_arrayvec 0.5.1 2020-09-01 +crate_arrayref 0.3.6 2020-09-01 +crate_autocfg 0.1.7 2020-09-01 +crate_autocfg 1.0.0 2020-09-01 +crate_bigint 4.4.2 2020-09-01 +crate_bigint 4.4.3 2020-09-01 +crate_blake2b_simd 0.5.9 2020-09-01 +crate_blake2b_simd 0.5.10 2020-09-01 +crate_blake2s_simd 0.5.9 2020-09-01 +crate_blake2s_simd 0.5.10 2020-09-01 +crate_bit_vec 0.5.0 2020-09-01 +crate_bit_vec 0.5.1 2020-09-01 +crate_bit_vec 0.6.0 2020-09-01 +crate_bit_vec 0.6.1 2020-09-01 +crate_bit_vec 0.6.2 2020-09-01 +crate_block_cipher_trait 0.7.0 2020-09-01 +crate_byteorder 1.3.3 2020-09-01 +crate_byteorder 1.3.4 2020-09-01 +crate_block_buffer 0.8.0 2020-09-01 +crate_block_buffer 0.9.0 2020-09-01 +crate_block_padding 0.1.5 2020-09-01 +crate_block_padding 0.2.0 2020-09-01 +crate_c2_chacha 0.2.3 2020-09-01 +crate_c2_chacha 0.2.4 2020-09-01 +crate_cfg_if 0.1.10 2020-09-01 +crate_crunchy 0.2.1 2020-09-01 +crate_crunchy 0.2.2 2020-09-01 +crate_constant_time_eq 0.1.5 2020-09-01 +crate_crossbeam 0.7.3 2020-09-01 +crate_digest 0.9.0 2020-09-01 +crate_crossbeam_channel 0.4.0 2020-09-01 +crate_crossbeam_channel 0.4.1 2020-09-01 +crate_crossbeam_channel 0.4.2 2020-09-01 +crate_crossbeam_deque 0.7.2 2020-09-01 +crate_crossbeam_deque 0.7.3 2020-09-01 +crate_crossbeam_epoch 0.8.0 2020-09-01 +crate_crossbeam_epoch 0.8.1 2020-09-01 +crate_crossbeam_epoch 0.8.2 2020-09-01 +crate_crossbeam_utils 0.7.0 2020-09-01 +crate_crossbeam_utils 0.7.1 2020-09-01 +crate_crossbeam_utils 0.7.2 2020-09-01 +crate_crossbeam_queue 0.2.0 2020-09-01 +crate_crossbeam_queue 0.2.1 2020-09-01 +crate_crossbeam_queue 0.2.2 2020-09-01 +crate_crossbeam_queue 0.2.3 2020-09-01 +crate_crypto_api_chachapoly 0.3.0 2020-09-01 +crate_crypto_api_chachapoly 0.4.0 2020-09-01 +crate_crypto_api_chachapoly 0.4.1 2020-09-01 +crate_crypto_api_chachapoly 0.4.2 2020-09-01 +crate_directories 2.0.0 2020-09-01 +crate_directories 2.0.1 2020-09-01 +crate_directories 2.0.2 2020-09-01 +crate_directories 3.0.0 2020-09-01 +crate_getrandom 0.1.13 2020-09-01 +crate_getrandom 0.1.14 2020-09-01 +crate_hex 0.4.0 2020-09-01 +crate_hex 0.4.1 2020-09-01 +crate_hex 0.4.2 2020-09-01 +crate_log 0.4.9 2020-09-01 +crate_log 0.4.10 2020-09-01 +crate_futures 0.2.0 2020-09-01 +crate_futures 0.2.1 2020-09-01 +crate_futures 0.3.0 2020-09-01 +crate_futures 0.3.1 2020-09-01 +crate_futures 0.3.2 2020-09-01 +crate_futures 0.3.3 2020-09-01 +crate_futures 0.3.4 2020-09-01 +crate_futures 0.3.5 2020-09-01 +crate_generic_array 0.13.0 2020-09-01 +crate_generic_array 0.13.1 2020-09-01 +crate_generic_array 0.13.2 2020-09-01 +crate_generic_array 0.14.0 2020-09-01 +crate_generic_array 0.14.1 2020-09-01 +crate_generic_array 0.14.2 2020-09-01 +crate_libc 0.2.63 2020-09-01 +crate_libc 0.2.64 2020-09-01 +crate_libc 0.2.65 2020-09-01 +crate_libc 0.2.66 2020-09-01 +crate_libc 0.2.67 2020-09-01 +crate_libc 0.2.68 2020-09-01 +crate_libc 0.2.69 2020-09-01 +crate_libc 0.2.70 2020-09-01 +crate_libc 0.2.71 2020-09-01 +crate_nodrop 0.1.14 2020-09-01 +crate_num_bigint 0.2.4 2020-09-01 +crate_num_bigint 0.2.5 2020-09-01 +crate_num_bigint 0.2.6 2020-09-01 +crate_num_bigint 0.3.0 2020-09-01 +crate_memoffset 0.5.2 2020-09-01 +crate_memoffset 0.5.3 2020-09-01 +crate_memoffset 0.5.4 2020-09-01 +crate_ppv_lite86 0.2.6 2020-09-01 +crate_ppv_lite86 0.2.7 2020-09-01 +crate_ppv_lite86 0.2.8 2020-09-01 +crate_proc_macro2 1.0.4 2020-09-01 +crate_proc_macro2 1.0.5 2020-09-01 +crate_proc_macro2 1.0.6 2020-09-01 +crate_proc_macro2 1.0.7 2020-09-01 +crate_proc_macro2 1.0.8 2020-09-01 +crate_proc_macro2 1.0.9 2020-09-01 +crate_proc_macro2 1.0.10 2020-09-01 +crate_proc_macro2 1.0.11 2020-09-01 +crate_proc_macro2 1.0.12 2020-09-01 +crate_proc_macro2 1.0.13 2020-09-01 +crate_proc_macro2 1.0.14 2020-09-01 +crate_proc_macro2 1.0.15 2020-09-01 +crate_proc_macro2 1.0.16 2020-09-01 +crate_proc_macro2 1.0.17 2020-09-01 +crate_proc_macro2 1.0.18 2020-09-01 +crate_quote 1.0.3 2020-09-01 +crate_quote 1.0.4 2020-09-01 +crate_quote 1.0.5 2020-09-01 +crate_quote 1.0.6 2020-09-01 +crate_quote 1.0.7 2020-09-01 +crate_num_cpus 1.11.0 2020-09-01 +crate_num_cpus 1.11.1 2020-09-01 +crate_num_cpus 1.12.0 2020-09-01 +crate_num_cpus 1.13.0 2020-09-01 +crate_num_integer 0.1.42 2020-09-01 +crate_num_integer 0.1.43 2020-09-01 +crate_num_traits 0.2.9 2020-09-01 +crate_num_traits 0.2.10 2020-09-01 +crate_num_traits 0.2.11 2020-09-01 +crate_num_traits 0.2.12 2020-09-01 +crate_opaque_debug 0.3.0 2020-09-01 +crate_rand 0.7.1 2020-09-01 +crate_rand 0.7.2 2020-09-01 +crate_rand 0.7.3 2020-09-01 +crate_typenum 1.12.0 2020-09-01 +crate_rand_chacha 0.2.2 2020-09-01 +crate_scopeguard 1.1.0 2020-09-01 +crate_semver 0.10.0 2020-09-01 +crate_semver_parser 0.9.0 2020-09-01 +crate_sha2 0.8.1 2020-09-01 +crate_sha2 0.8.2 2020-09-01 +crate_sha2 0.9.0 2020-09-01 +crate_sha2 0.9.1 2020-09-01 +crate_syn 1.0.6 2020-09-01 +crate_syn 1.0.7 2020-09-01 +crate_syn 1.0.8 2020-09-01 +crate_syn 1.0.9 2020-09-01 +crate_syn 1.0.10 2020-09-01 +crate_syn 1.0.11 2020-09-01 +crate_syn 1.0.12 2020-09-01 +crate_syn 1.0.13 2020-09-01 +crate_syn 1.0.14 2020-09-01 +crate_syn 1.0.15 2020-09-01 +crate_syn 1.0.16 2020-09-01 +crate_syn 1.0.17 2020-09-01 +crate_syn 1.0.18 2020-09-01 +crate_syn 1.0.19 2020-09-01 +crate_syn 1.0.20 2020-09-01 +crate_syn 1.0.21 2020-09-01 +crate_syn 1.0.22 2020-09-01 +crate_syn 1.0.23 2020-09-01 +crate_syn 1.0.24 2020-09-01 +crate_syn 1.0.25 2020-09-01 +crate_syn 1.0.26 2020-09-01 +crate_syn 1.0.27 2020-09-01 +crate_syn 1.0.28 2020-09-01 +crate_syn 1.0.29 2020-09-01 +crate_syn 1.0.30 2020-09-01 +crate_syn 1.0.31 2020-09-01 +crate_syn 1.0.32 2020-09-01 +crate_syn 1.0.33 2020-09-01 +crate_unicode_xid 0.2.1 2020-09-01 \ No newline at end of file diff --git a/qa/zcash/updatecheck.py b/qa/zcash/updatecheck.py index 59d0aa66a..4746a05b7 100755 --- a/qa/zcash/updatecheck.py +++ b/qa/zcash/updatecheck.py @@ -30,6 +30,7 @@ import requests import os import re import sys +import datetime SOURCE_ROOT = os.path.join(os.path.dirname(os.path.realpath(__file__)), "..", "..") # The email for this account is taylor@electriccoin.co and the token does not @@ -258,7 +259,7 @@ class BerkeleyDbReleaseLister: for match in re.findall("Berkeley DB (\d+)\.(\d+)\.(\d+)\.tar.gz", page): release_versions.add(Version(match)) - if Version((6, 2, 38)) not in release_versions: + if len(release_versions) == 0: raise RuntimeError("Missing expected version from Oracle web page.") return list(release_versions) @@ -312,6 +313,33 @@ class UnivalueVersionGetter: else: raise RuntimeError("Couldn't parse univalue's version from its configure.ac") +class PostponedUpdates(): + def __init__(self): + self.postponedlist = dict() + + postponedlist_path = os.path.join( + os.path.dirname(__file__), + "postponed-updates.txt" + ) + + file = open(postponedlist_path, 'r') + for line in file.readlines(): + stripped = re.sub('#.*$', '', line).strip() + if stripped != "": + match = re.match('^(\S+)\s+(\S+)\s+(\S+)$', stripped) + if match: + postponed_name = match.groups()[0] + postponed_version = Version(match.groups()[1].split(".")) + postpone_expiration = datetime.datetime.strptime(match.groups()[2], '%Y-%m-%d') + if datetime.datetime.utcnow() < postpone_expiration: + self.postponedlist[(postponed_name, str(postponed_version))] = True + else: + raise RuntimeError("Could not parse line in postponed-updates.txt:" + line) + + + def is_postponed(self, name, version): + return (name, str(version)) in self.postponedlist + def safe(string): if re.match('^[a-zA-Z0-9_-]*$', string): return string @@ -356,6 +384,7 @@ def main(): sys.exit(status) deps = get_dependency_list() + postponed = PostponedUpdates() for dependency in deps: if dependency.name in unchecked_dependencies: unchecked_dependencies.remove(dependency.name) @@ -369,17 +398,36 @@ def main(): str(dependency.current_version()), "") else: + # The status can either be POSTPONED or OUT OF DATE depending + # on whether or not all the new versions are whitelisted. + status_text = "POSTPONED" + newver_list = "[" + for newver in dependency.released_versions_after_current_version(): + if postponed.is_postponed(dependency.name, newver): + newver_list += str(newver) + " (postponed)," + else: + newver_list += str(newver) + "," + status_text = "OUT OF DATE" + status = 1 + + newver_list = newver_list[:-1] + "]" + print_row( dependency.name, - "OUT OF DATE", + status_text, str(dependency.current_version()), - str(list(map(str, dependency.released_versions_after_current_version())))) - status = 1 + newver_list + ) if len(unchecked_dependencies) > 0: unchecked_dependencies.sort() print("WARNING: The following dependencies are not being checked for updates by this script: " + ', '.join(unchecked_dependencies)) - status = 2 + sys.exit(2) + + if status == 0: + print("Ready to release. All dependencies are up-to-date or postponed.") + elif status == 1: + print("Release is BLOCKED. There are new dependency updates that have not been postponed.") sys.exit(status) From 44c35c6f68beac02fcda8e1f13650d74f4224a91 Mon Sep 17 00:00:00 2001 From: Taylor Hornby Date: Wed, 24 Jun 2020 12:21:42 -0600 Subject: [PATCH 084/725] Change release instructions to block the release when dependencies are not updated and not postponed. --- doc/release-process.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/doc/release-process.md b/doc/release-process.md index 44d43364f..3bfec0b3a 100644 --- a/doc/release-process.md +++ b/doc/release-process.md @@ -26,6 +26,16 @@ Ensure that new performance metrics appear on that site. Update `src/chainparams.cpp` nMinimumChainWork with information from the getblockchaininfo rpc. +Check that dependencies are up-to-date or have been postponed: + +``` +$ ./qa/zcash/updatecheck.py +``` + +If there are updates that have not been postponed, review their changelogs +for urgent security fixes, and if there aren't any, postpone the update by +adding a line to `qa/zcash/postponed-updates.txt`. + ### Protocol Safety Checks: If this release changes the behavior of the protocol or fixes a serious From 8f5dbd32932a0e99e0611dd5411d22d7c2c1c539 Mon Sep 17 00:00:00 2001 From: Taylor Hornby Date: Wed, 24 Jun 2020 12:31:56 -0600 Subject: [PATCH 085/725] Enforce pre-release dependency update check in make-release.py --- zcutil/make-release.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zcutil/make-release.py b/zcutil/make-release.py index 53b512702..572072bb3 100755 --- a/zcutil/make-release.py +++ b/zcutil/make-release.py @@ -84,6 +84,7 @@ def main_logged(release, releaseprev, releasefrom, releaseheight, hotfix): verify_tags(releaseprev, releasefrom) verify_version(release, releaseprev, hotfix) + verify_dependency_updates() initialize_git(release, hotfix) patch_version_in_files(release, releaseprev) patch_release_height(releaseheight) @@ -125,6 +126,11 @@ def verify_dependencies(dependencies): ), ) +@phase('Checking dependency updates.') +def verify_dependency_updates(): + status = subprocess.call(['python', 'qa/zcash/updatecheck.py']) + if status != 0: + raise SystemExit("Dependency update check did not pass.") @phase('Checking tags.') def verify_tags(releaseprev, releasefrom): From 4af761121d6d5743a281c56e0ee48263b048dedd Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Wed, 17 Jun 2020 16:10:13 -0600 Subject: [PATCH 086/725] Require that shielded coinbase output note plaintexts are version 2 if Canopy is active. Co-authored by Ying Tong (yingtong@electriccoin.co) --- src/main.cpp | 17 ++++++++++++++--- src/zcash/Note.hpp | 3 +++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index d11ca5bae..cbdd8a18d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -792,6 +792,7 @@ bool ContextualCheckTransaction( bool saplingActive = chainparams.GetConsensus().NetworkUpgradeActive(nHeight, Consensus::UPGRADE_SAPLING); bool isSprout = !overwinterActive; bool heartwoodActive = chainparams.GetConsensus().NetworkUpgradeActive(nHeight, Consensus::UPGRADE_HEARTWOOD); + bool canopyActive = chainparams.GetConsensus().NetworkUpgradeActive(nHeight, Consensus::UPGRADE_CANOPY); // If Sprout rules apply, reject transactions which are intended for Overwinter and beyond if (isSprout && tx.fOverwintered) { @@ -919,19 +920,29 @@ bool ContextualCheckTransaction( } // SaplingNotePlaintext::decrypt() checks note commitment validity. - if (!SaplingNotePlaintext::decrypt( + auto encPlaintext = SaplingNotePlaintext::decrypt( output.encCiphertext, output.ephemeralKey, outPlaintext->esk, outPlaintext->pk_d, - output.cmu) - ) { + output.cmu); + if (!encPlaintext) { return state.DoS( DOS_LEVEL_BLOCK, error("CheckTransaction(): coinbase output description has invalid encCiphertext"), REJECT_INVALID, "bad-cb-output-desc-invalid-encct"); } + + // ZIP 212: Check that the note plaintexts use the v2 note plaintext + // version. + if (canopyActive != (encPlaintext->get_lead_byte() == 0x02)) { + return state.DoS( + DOS_LEVEL_BLOCK, + error("CheckTransaction(): coinbase output description has invalid note plaintext version"), + REJECT_INVALID, + "bad-cb-output-desc-invalid-note-plaintext-version"); + } } } } diff --git a/src/zcash/Note.hpp b/src/zcash/Note.hpp index a130267a7..a56a2310e 100644 --- a/src/zcash/Note.hpp +++ b/src/zcash/Note.hpp @@ -171,6 +171,9 @@ public: uint256 rcm() const; uint256 generate_esk() const; + bool get_lead_byte() const { + return leadByte; + } }; class SaplingOutgoingPlaintext From 56d4ef8333a5c9db04aee636ec383b85089902ee Mon Sep 17 00:00:00 2001 From: Sean Bowe Date: Wed, 17 Jun 2020 17:09:11 -0600 Subject: [PATCH 087/725] Make transaction builder take the next block height into account for use of v2 note plaintexts. --- src/transaction_builder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transaction_builder.cpp b/src/transaction_builder.cpp index 7390932af..52286d00c 100644 --- a/src/transaction_builder.cpp +++ b/src/transaction_builder.cpp @@ -163,7 +163,7 @@ void TransactionBuilder::AddSaplingOutput( } unsigned char leadByte = 0x01; - if (Params().GetConsensus().NetworkUpgradeActive(nHeight, Consensus::UPGRADE_CANOPY)) { + if (Params().GetConsensus().NetworkUpgradeActive(nHeight + 1, Consensus::UPGRADE_CANOPY)) { leadByte = 0x02; } auto note = libzcash::SaplingNote(to, value, leadByte); From e060d598901f815d06b35debbad4a6a6d3ca0834 Mon Sep 17 00:00:00 2001 From: therealyingtong Date: Fri, 19 Jun 2020 11:10:19 +0800 Subject: [PATCH 088/725] Reject v1 plaintexts after grace period SaplingNotePlaintext::decrypt() now has to be aware of consensus params and blockheight. Its callers in wallet, rpcwallet, and tests are updated accordingly. TransactionBuilder is also modified to reject invalid leadBytes. Co-authored by Daira Hopwood (daira@jacaranda.org) --- src/consensus/consensus.h | 2 ++ src/gtest/test_noteencryption.cpp | 17 ++++++++++ src/main.cpp | 2 ++ src/transaction_builder.cpp | 6 ++++ src/wallet/gtest/test_wallet.cpp | 7 ++-- src/wallet/rpcwallet.cpp | 10 ++++-- src/wallet/wallet.cpp | 20 +++++++++--- src/wallet/wallet.h | 4 +-- src/zcash/Note.cpp | 53 +++++++++++++++---------------- src/zcash/Note.hpp | 41 +++++++++++++++++++----- 10 files changed, 114 insertions(+), 48 deletions(-) diff --git a/src/consensus/consensus.h b/src/consensus/consensus.h index 847e5d53a..3452cd4b4 100644 --- a/src/consensus/consensus.h +++ b/src/consensus/consensus.h @@ -31,6 +31,8 @@ static const unsigned int MAX_TX_SIZE_AFTER_SAPLING = MAX_BLOCK_SIZE; static const int COINBASE_MATURITY = 100; /** The minimum value which is invalid for expiry height, used by CTransaction and CMutableTransaction */ static constexpr uint32_t TX_EXPIRY_HEIGHT_THRESHOLD = 500000000; +/** The number of blocks after Canopy activation after which v1 plaintexts will be rejected */ +static const unsigned int ZIP212_GRACE_PERIOD = 32256; /** Flags for LockTime() */ enum { diff --git a/src/gtest/test_noteencryption.cpp b/src/gtest/test_noteencryption.cpp index 083bdeba1..2d99158b2 100644 --- a/src/gtest/test_noteencryption.cpp +++ b/src/gtest/test_noteencryption.cpp @@ -10,6 +10,8 @@ #include "zcash/Address.hpp" #include "crypto/sha256.h" #include "librustzcash.h" +#include "consensus/params.h" +#include "utiltest.h" class TestNoteDecryption : public ZCNoteDecryption { public: @@ -22,6 +24,13 @@ public: TEST(noteencryption, NotePlaintext) { + SelectParams(CBaseChainParams::REGTEST); + const Consensus::Params& params = Params().GetConsensus(); + int overwinterActivationHeight = 5; + int saplingActivationHeight = 30; + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, overwinterActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, saplingActivationHeight); + using namespace libzcash; auto xsk = SaplingSpendingKey(uint256()).expanded_spending_key(); auto fvk = xsk.full_viewing_key(); @@ -55,6 +64,8 @@ TEST(noteencryption, NotePlaintext) // Try to decrypt with incorrect commitment ASSERT_FALSE(SaplingNotePlaintext::decrypt( + params, + saplingActivationHeight, ct, ivk, epk, @@ -63,6 +74,8 @@ TEST(noteencryption, NotePlaintext) // Try to decrypt with correct commitment auto foo = SaplingNotePlaintext::decrypt( + params, + saplingActivationHeight, ct, ivk, epk, @@ -129,6 +142,8 @@ TEST(noteencryption, NotePlaintext) // Test sender won't accept invalid commitments ASSERT_FALSE( SaplingNotePlaintext::decrypt( + params, + saplingActivationHeight, ct, epk, decrypted_out_ct_unwrapped.esk, @@ -139,6 +154,8 @@ TEST(noteencryption, NotePlaintext) // Test sender can decrypt the note ciphertext. foo = SaplingNotePlaintext::decrypt( + params, + saplingActivationHeight, ct, epk, decrypted_out_ct_unwrapped.esk, diff --git a/src/main.cpp b/src/main.cpp index cbdd8a18d..8e2ebaed9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -921,6 +921,8 @@ bool ContextualCheckTransaction( // SaplingNotePlaintext::decrypt() checks note commitment validity. auto encPlaintext = SaplingNotePlaintext::decrypt( + chainparams.GetConsensus(), + nHeight, output.encCiphertext, output.ephemeralKey, outPlaintext->esk, diff --git a/src/transaction_builder.cpp b/src/transaction_builder.cpp index 52286d00c..1d6eddbb5 100644 --- a/src/transaction_builder.cpp +++ b/src/transaction_builder.cpp @@ -9,6 +9,7 @@ #include "rpc/protocol.h" #include "script/sign.h" #include "utilmoneystr.h" +#include "zcash/Note.hpp" #include #include @@ -142,6 +143,11 @@ void TransactionBuilder::AddSaplingSpend( throw std::runtime_error("TransactionBuilder cannot add Sapling spend to pre-Sapling transaction"); } + // ZIP212: check that note plaintext lead byte is valid at height + if (!libzcash::plaintext_version_is_valid(consensusParams, nHeight + 1, note.get_lead_byte())) { + throw std::runtime_error("TransactionBuilder: invalid note plaintext version"); + } + // Consistency check: all anchors must equal the first one if (spends.size() > 0 && spends[0].anchor != anchor) { throw JSONRPCError(RPC_WALLET_ERROR, "Anchor does not match previously-added Sapling spends."); diff --git a/src/wallet/gtest/test_wallet.cpp b/src/wallet/gtest/test_wallet.cpp index 18865cbcd..348b2e022 100644 --- a/src/wallet/gtest/test_wallet.cpp +++ b/src/wallet/gtest/test_wallet.cpp @@ -698,6 +698,8 @@ TEST(WalletTests, GetConflictedSaplingNotes) { // Decrypt output note B auto maybe_pt = libzcash::SaplingNotePlaintext::decrypt( + consensusParams, + wtx.nExpiryHeight, wtx.vShieldedOutput[0].encCiphertext, ivk, wtx.vShieldedOutput[0].ephemeralKey, @@ -1075,6 +1077,8 @@ TEST(WalletTests, SpentSaplingNoteIsFromMe) { // Decrypt note B auto maybe_pt = libzcash::SaplingNotePlaintext::decrypt( + consensusParams, + wtx.nExpiryHeight, wtx.vShieldedOutput[0].encCiphertext, ivk, wtx.vShieldedOutput[0].ephemeralKey, @@ -2005,8 +2009,7 @@ TEST(WalletTests, MarkAffectedSaplingTransactionsDirty) { wtx = wallet.mapWallet[hash]; // Prepare to spend the note that was just created - auto maybe_pt = libzcash::SaplingNotePlaintext::decrypt( - tx1.vShieldedOutput[0].encCiphertext, ivk, tx1.vShieldedOutput[0].ephemeralKey, tx1.vShieldedOutput[0].cmu); + auto maybe_pt = libzcash::SaplingNotePlaintext::decrypt(consensusParams, fakeIndex.nHeight, tx1.vShieldedOutput[0].encCiphertext, ivk, tx1.vShieldedOutput[0].ephemeralKey, tx1.vShieldedOutput[0].cmu); ASSERT_EQ(static_cast(maybe_pt), true); auto maybe_note = maybe_pt.get().note(ivk); ASSERT_EQ(static_cast(maybe_note), true); diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index 36838401d..e45aa218e 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -5,6 +5,7 @@ #include "amount.h" #include "consensus/upgrades.h" +#include "consensus/params.h" #include "core_io.h" #include "experimental_features.h" #include "init.h" @@ -3768,7 +3769,8 @@ UniValue z_viewtransaction(const UniValue& params, bool fHelp) auto op = res->second; auto wtxPrev = pwalletMain->mapWallet.at(op.hash); - auto decrypted = wtxPrev.DecryptSaplingNote(op).get(); + // TODO: decide which height to use here instead of wtxPrev.nExpiryHeight + auto decrypted = wtxPrev.DecryptSaplingNote(Params().GetConsensus(), wtxPrev.nExpiryHeight, op).get(); auto notePt = decrypted.first; auto pa = decrypted.second; @@ -3796,14 +3798,16 @@ UniValue z_viewtransaction(const UniValue& params, bool fHelp) SaplingPaymentAddress pa; bool isOutgoing; - auto decrypted = wtx.DecryptSaplingNote(op); + // TODO: decide which height to use here instead of wtx.nExpiryHeight + auto decrypted = wtx.DecryptSaplingNote(Params().GetConsensus(), wtx.nExpiryHeight, op); if (decrypted) { notePt = decrypted->first; pa = decrypted->second; isOutgoing = false; } else { // Try recovering the output - auto recovered = wtx.RecoverSaplingNote(op, ovks); + // TODO: decide which height to use here instead of wtxPrev.nExpiryHeight + auto recovered = wtx.RecoverSaplingNote(Params().GetConsensus(), wtx.nExpiryHeight, op, ovks); if (recovered) { notePt = recovered->first; pa = recovered->second; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index a1c7bb5c0..0ccea2ae0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1495,7 +1495,9 @@ void CWallet::UpdateSaplingNullifierNoteMapWithTx(CWalletTx& wtx) { uint64_t position = nd.witnesses.front().position(); auto extfvk = mapSaplingFullViewingKeys.at(nd.ivk); OutputDescription output = wtx.vShieldedOutput[op.n]; - auto optPlaintext = SaplingNotePlaintext::decrypt(output.encCiphertext, nd.ivk, output.ephemeralKey, output.cmu); + + // TODO: decide which height to use here instead of wtx.nExpiryHeight + auto optPlaintext = SaplingNotePlaintext::decrypt(Params().GetConsensus(), wtx.nExpiryHeight, output.encCiphertext, nd.ivk, output.ephemeralKey, output.cmu); if (!optPlaintext) { // An item in mapSaplingNoteData must have already been successfully decrypted, // otherwise the item would not exist in the first place. @@ -1901,7 +1903,9 @@ std::pair CWallet::FindMySap const OutputDescription output = tx.vShieldedOutput[i]; for (auto it = mapSaplingFullViewingKeys.begin(); it != mapSaplingFullViewingKeys.end(); ++it) { SaplingIncomingViewingKey ivk = it->first; - auto result = SaplingNotePlaintext::decrypt(output.encCiphertext, ivk, output.ephemeralKey, output.cmu); + + // TODO: decide which height to use here instead of wtx.nExpiryHeight + auto result = SaplingNotePlaintext::decrypt(Params().GetConsensus(), tx.nExpiryHeight, output.encCiphertext, ivk, output.ephemeralKey, output.cmu); if (!result) { continue; } @@ -2300,7 +2304,7 @@ std::pair CWalletTx::DecryptSproutNot boost::optional> CWalletTx::DecryptSaplingNote(SaplingOutPoint op) const + SaplingPaymentAddress>> CWalletTx::DecryptSaplingNote(const Consensus::Params& params, int height, SaplingOutPoint op) const { // Check whether we can decrypt this SaplingOutPoint if (this->mapSaplingNoteData.count(op) == 0) { @@ -2311,6 +2315,8 @@ boost::optionalmapSaplingNoteData.at(op); auto maybe_pt = SaplingNotePlaintext::decrypt( + params, + height, output.encCiphertext, nd.ivk, output.ephemeralKey, @@ -2327,8 +2333,7 @@ boost::optional> CWalletTx::RecoverSaplingNote( - SaplingOutPoint op, std::set& ovks) const + SaplingPaymentAddress>> CWalletTx::RecoverSaplingNote(const Consensus::Params& params, int height, SaplingOutPoint op, std::set& ovks) const { auto output = this->vShieldedOutput[op.n]; @@ -2344,6 +2349,8 @@ boost::optionalesk, @@ -4975,7 +4982,10 @@ void CWallet::GetFilteredNotes( SaplingOutPoint op = pair.first; SaplingNoteData nd = pair.second; + // TODO: decide which height to use here instead of wtx.nExpiryHeight auto maybe_pt = SaplingNotePlaintext::decrypt( + Params().GetConsensus(), + wtx.nExpiryHeight, wtx.vShieldedOutput[op.n].encCiphertext, nd.ivk, wtx.vShieldedOutput[op.n].ephemeralKey, diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index c673dfcea..d0696afd6 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -566,10 +566,10 @@ public: JSOutPoint jsop) const; boost::optional> DecryptSaplingNote(SaplingOutPoint op) const; + libzcash::SaplingPaymentAddress>> DecryptSaplingNote(const Consensus::Params& params, int height, SaplingOutPoint op) const; boost::optional> RecoverSaplingNote( + libzcash::SaplingPaymentAddress>> RecoverSaplingNote(const Consensus::Params& params, int height, SaplingOutPoint op, std::set& ovks) const; //! filter decides which addresses will count towards the debit diff --git a/src/zcash/Note.cpp b/src/zcash/Note.cpp index 76a14f9e6..206b73cd9 100644 --- a/src/zcash/Note.cpp +++ b/src/zcash/Note.cpp @@ -1,6 +1,7 @@ #include "Note.hpp" #include "prf.h" #include "crypto/sha256.h" +#include "consensus/consensus.h" #include "random.h" #include "version.h" @@ -188,24 +189,20 @@ boost::optional SaplingOutgoingPlaintext::decrypt( } // Deserialize from the plaintext - try { - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << pt.get(); + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << pt.get(); - SaplingOutgoingPlaintext ret; - ss >> ret; + SaplingOutgoingPlaintext ret; + ss >> ret; - assert(ss.size() == 0); + assert(ss.size() == 0); - return ret; - } catch (const boost::thread_interrupted&) { - throw; - } catch (...) { - return boost::none; - } + return ret; } boost::optional SaplingNotePlaintext::decrypt( + const Consensus::Params& params, + int height, const SaplingEncCiphertext &ciphertext, const uint256 &ivk, const uint256 &epk, @@ -219,14 +216,13 @@ boost::optional SaplingNotePlaintext::decrypt( // Deserialize from the plaintext SaplingNotePlaintext ret; - try { - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << pt.get(); - ss >> ret; - assert(ss.size() == 0); - } catch (const boost::thread_interrupted&) { - throw; - } catch (...) { + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << pt.get(); + ss >> ret; + assert(ss.size() == 0); + + // Check leadbyte is allowed at block height + if (!plaintext_version_valid(params, height, ret.leadByte)) { return boost::none; } @@ -269,6 +265,8 @@ boost::optional SaplingNotePlaintext::decrypt( } boost::optional SaplingNotePlaintext::decrypt( + const Consensus::Params& params, + int height, const SaplingEncCiphertext &ciphertext, const uint256 &epk, const uint256 &esk, @@ -283,14 +281,13 @@ boost::optional SaplingNotePlaintext::decrypt( // Deserialize from the plaintext SaplingNotePlaintext ret; - try { - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << pt.get(); - ss >> ret; - assert(ss.size() == 0); - } catch (const boost::thread_interrupted&) { - throw; - } catch (...) { + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << pt.get(); + ss >> ret; + assert(ss.size() == 0); + + // Check leadbyte is legible at block height + if (!plaintext_version_valid(params, height, ret.leadByte)) { return boost::none; } diff --git a/src/zcash/Note.hpp b/src/zcash/Note.hpp index a56a2310e..134c289eb 100644 --- a/src/zcash/Note.hpp +++ b/src/zcash/Note.hpp @@ -5,6 +5,8 @@ #include "Zcash.h" #include "Address.hpp" #include "NoteEncryption.hpp" +#include "consensus/params.h" +#include "consensus/consensus.h" #include #include @@ -40,6 +42,27 @@ public: uint256 nullifier(const SproutSpendingKey& a_sk) const; }; +inline bool plaintext_version_is_valid(const Consensus::Params& params, int height, unsigned char leadByte) { + int canopyActivationHeight = params.vUpgrades[Consensus::UPGRADE_CANOPY].nActivationHeight; + + if (height < canopyActivationHeight && leadByte != 0x01) { + // non-0x01 received before Canopy activation height + return false; + } + if (height >= canopyActivationHeight + && height < canopyActivationHeight + ZIP212_GRACE_PERIOD + && leadByte != 0x01 + && leadByte != 0x02) + { + // non-{0x01,0x02} received after Canopy activation and before grace period has elapsed + return false; + } + if (height >= canopyActivationHeight + ZIP212_GRACE_PERIOD && leadByte != 0x02) { + // non-0x02 received past (Canopy activation height + grace period) + return false; + } + return true; +}; class SaplingNote : public BaseNote { private: @@ -60,6 +83,10 @@ public: boost::optional cmu() const; boost::optional nullifier(const SaplingFullViewingKey &vk, const uint64_t position) const; uint256 rcm() const; + + unsigned char get_lead_byte() const { + return leadByte; + } }; class BaseNotePlaintext { @@ -132,6 +159,8 @@ public: SaplingNotePlaintext(const SaplingNote& note, std::array memo); static boost::optional decrypt( + const Consensus::Params& params, + int height, const SaplingEncCiphertext &ciphertext, const uint256 &ivk, const uint256 &epk, @@ -139,6 +168,8 @@ public: ); static boost::optional decrypt( + const Consensus::Params& params, + int height, const SaplingEncCiphertext &ciphertext, const uint256 &epk, const uint256 &esk, @@ -154,13 +185,7 @@ public: template inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(leadByte); - - if (leadByte != 0x01 && leadByte != 0x02) { - printf("leadByte: %x\n", leadByte); - throw std::ios_base::failure("lead byte of SaplingNotePlaintext is not recognized"); - } - + READWRITE(leadByte); // 1 byte READWRITE(d); // 11 bytes READWRITE(value_); // 8 bytes READWRITE(rseed); // 32 bytes @@ -171,7 +196,7 @@ public: uint256 rcm() const; uint256 generate_esk() const; - bool get_lead_byte() const { + unsigned char get_lead_byte() const { return leadByte; } }; From 3c8e9703589bedf5c1d17438a9f1701d4fd7035c Mon Sep 17 00:00:00 2001 From: therealyingtong Date: Fri, 19 Jun 2020 11:09:27 +0800 Subject: [PATCH 089/725] Check epk vs esk whenever caller has esk --- src/zcash/Note.cpp | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/zcash/Note.cpp b/src/zcash/Note.cpp index 206b73cd9..76fcb1be2 100644 --- a/src/zcash/Note.cpp +++ b/src/zcash/Note.cpp @@ -222,7 +222,7 @@ boost::optional SaplingNotePlaintext::decrypt( assert(ss.size() == 0); // Check leadbyte is allowed at block height - if (!plaintext_version_valid(params, height, ret.leadByte)) { + if (!plaintext_version_is_valid(params, height, ret.leadByte)) { return boost::none; } @@ -287,7 +287,16 @@ boost::optional SaplingNotePlaintext::decrypt( assert(ss.size() == 0); // Check leadbyte is legible at block height - if (!plaintext_version_valid(params, height, ret.leadByte)) { + if (!plaintext_version_is_valid(params, height, ret.leadByte)) { + return boost::none; + } + + // Check that epk is consistent with esk + uint256 expected_epk; + if (!librustzcash_sapling_ka_derivepublic(ret.d.data(), esk.begin(), expected_epk.begin())) { + return boost::none; + } + if (expected_epk != epk) { return boost::none; } @@ -309,16 +318,6 @@ boost::optional SaplingNotePlaintext::decrypt( } if (ret.leadByte == 0x02) { - // ZIP 212: Check that epk is consistent to prevent against linkability - // attacks without relying on the soundness of the SNARK. - uint256 expected_epk; - if (!librustzcash_sapling_ka_derivepublic(ret.d.data(), esk.begin(), expected_epk.begin())) { - return boost::none; - } - if (expected_epk != epk) { - return boost::none; - } - // ZIP 212: Additionally check that the esk provided to this function // is consistent with the esk we can derive if (esk != ret.generate_esk()) { From 6402c589c6cb00185e14d4f8c4b6bb9acc981489 Mon Sep 17 00:00:00 2001 From: therealyingtong Date: Sat, 20 Jun 2020 15:35:16 +0800 Subject: [PATCH 090/725] Refactor SaplingNotePlaintext::decrypt Break up plaintext decryption into height-dependent and non-height-dependent parts. --- src/wallet/rpcwallet.cpp | 13 ++-- src/wallet/wallet.cpp | 110 +++++++++++++++++++++++++++--- src/wallet/wallet.h | 8 ++- src/zcash/Note.cpp | 144 +++++++++++++++++++++++++++------------ src/zcash/Note.hpp | 28 ++++++++ src/zcbenchmarks.cpp | 2 +- 6 files changed, 243 insertions(+), 62 deletions(-) diff --git a/src/wallet/rpcwallet.cpp b/src/wallet/rpcwallet.cpp index e45aa218e..08bf81662 100644 --- a/src/wallet/rpcwallet.cpp +++ b/src/wallet/rpcwallet.cpp @@ -3769,8 +3769,9 @@ UniValue z_viewtransaction(const UniValue& params, bool fHelp) auto op = res->second; auto wtxPrev = pwalletMain->mapWallet.at(op.hash); - // TODO: decide which height to use here instead of wtxPrev.nExpiryHeight - auto decrypted = wtxPrev.DecryptSaplingNote(Params().GetConsensus(), wtxPrev.nExpiryHeight, op).get(); + // We don't need to check the leadbyte here: if wtx exists in + // the wallet, it must have already passed the leadbyte check + auto decrypted = wtxPrev.DecryptSaplingNoteWithoutLeadByteCheck(op).get(); auto notePt = decrypted.first; auto pa = decrypted.second; @@ -3798,16 +3799,16 @@ UniValue z_viewtransaction(const UniValue& params, bool fHelp) SaplingPaymentAddress pa; bool isOutgoing; - // TODO: decide which height to use here instead of wtx.nExpiryHeight - auto decrypted = wtx.DecryptSaplingNote(Params().GetConsensus(), wtx.nExpiryHeight, op); + // We don't need to check the leadbyte here: if wtx exists in + // the wallet, it must have already passed the leadbyte check + auto decrypted = wtx.DecryptSaplingNoteWithoutLeadByteCheck(op); if (decrypted) { notePt = decrypted->first; pa = decrypted->second; isOutgoing = false; } else { // Try recovering the output - // TODO: decide which height to use here instead of wtxPrev.nExpiryHeight - auto recovered = wtx.RecoverSaplingNote(Params().GetConsensus(), wtx.nExpiryHeight, op, ovks); + auto recovered = wtx.RecoverSaplingNoteWithoutLeadByteCheck(op, ovks); if (recovered) { notePt = recovered->first; pa = recovered->second; diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 0ccea2ae0..d43881256 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1496,8 +1496,15 @@ void CWallet::UpdateSaplingNullifierNoteMapWithTx(CWalletTx& wtx) { auto extfvk = mapSaplingFullViewingKeys.at(nd.ivk); OutputDescription output = wtx.vShieldedOutput[op.n]; - // TODO: decide which height to use here instead of wtx.nExpiryHeight - auto optPlaintext = SaplingNotePlaintext::decrypt(Params().GetConsensus(), wtx.nExpiryHeight, output.encCiphertext, nd.ivk, output.ephemeralKey, output.cmu); + auto optDeserialized = SaplingNotePlaintext::attempt_sapling_enc_decryption_deserialization(output.encCiphertext, nd.ivk, output.ephemeralKey); + + if (!optDeserialized) { + // The transaction would not have entered the wallet unless + // its plaintest had been succesfully decrypted previously. + assert(false); + } + + auto optPlaintext = SaplingNotePlaintext::plaintext_checks_without_height(*optDeserialized, nd.ivk, output.ephemeralKey, output.cmu); if (!optPlaintext) { // An item in mapSaplingNoteData must have already been successfully decrypted, // otherwise the item would not exist in the first place. @@ -1716,7 +1723,7 @@ bool CWallet::AddToWalletIfInvolvingMe(const CTransaction& tx, const CBlock* pbl bool fExisted = mapWallet.count(tx.GetHash()) != 0; if (fExisted && !fUpdate) return false; auto sproutNoteData = FindMySproutNotes(tx); - auto saplingNoteDataAndAddressesToAdd = FindMySaplingNotes(tx); + auto saplingNoteDataAndAddressesToAdd = FindMySaplingNotes(tx, chainActive.Height()); auto saplingNoteData = saplingNoteDataAndAddressesToAdd.first; auto addressesToAdd = saplingNoteDataAndAddressesToAdd.second; for (const auto &addressToAdd : addressesToAdd) { @@ -1890,7 +1897,7 @@ mapSproutNoteData_t CWallet::FindMySproutNotes(const CTransaction &tx) const * the result of FindMySaplingNotes (for the addresses available at the time) will * already have been cached in CWalletTx.mapSaplingNoteData. */ -std::pair CWallet::FindMySaplingNotes(const CTransaction &tx) const +std::pair CWallet::FindMySaplingNotes(const CTransaction &tx, int height) const { LOCK(cs_KeyStore); uint256 hash = tx.GetHash(); @@ -1904,8 +1911,7 @@ std::pair CWallet::FindMySap for (auto it = mapSaplingFullViewingKeys.begin(); it != mapSaplingFullViewingKeys.end(); ++it) { SaplingIncomingViewingKey ivk = it->first; - // TODO: decide which height to use here instead of wtx.nExpiryHeight - auto result = SaplingNotePlaintext::decrypt(Params().GetConsensus(), tx.nExpiryHeight, output.encCiphertext, ivk, output.ephemeralKey, output.cmu); + auto result = SaplingNotePlaintext::decrypt(Params().GetConsensus(), height, output.encCiphertext, ivk, output.ephemeralKey, output.cmu); if (!result) { continue; } @@ -2331,6 +2337,41 @@ boost::optional> CWalletTx::DecryptSaplingNoteWithoutLeadByteCheck(SaplingOutPoint op) const +{ + // Check whether we can decrypt this SaplingOutPoint + if (this->mapSaplingNoteData.count(op) == 0) { + return boost::none; + } + + auto output = this->vShieldedOutput[op.n]; + auto nd = this->mapSaplingNoteData.at(op); + + auto optDeserialized = SaplingNotePlaintext::attempt_sapling_enc_decryption_deserialization(output.encCiphertext, nd.ivk, output.ephemeralKey); + + if (!optDeserialized) { + // The transaction would not have entered the wallet unless + // its plaintest had been succesfully decrypted previously. + assert(false); + } + + auto maybe_pt = SaplingNotePlaintext::plaintext_checks_without_height( + *optDeserialized, + nd.ivk, + output.ephemeralKey, + output.cmu); + assert(static_cast(maybe_pt)); + auto notePt = maybe_pt.get(); + + auto maybe_pa = nd.ivk.address(notePt.d); + assert(static_cast(maybe_pa)); + auto pa = maybe_pa.get(); + + return std::make_pair(notePt, pa); +} + boost::optional> CWalletTx::RecoverSaplingNote(const Consensus::Params& params, int height, SaplingOutPoint op, std::set& ovks) const @@ -2366,6 +2407,47 @@ boost::optional> CWalletTx::RecoverSaplingNoteWithoutLeadByteCheck(SaplingOutPoint op, std::set& ovks) const +{ + auto output = this->vShieldedOutput[op.n]; + + for (auto ovk : ovks) { + auto outPt = SaplingOutgoingPlaintext::decrypt( + output.outCiphertext, + ovk, + output.cv, + output.cmu, + output.ephemeralKey); + if (!outPt) { + continue; + } + + auto optDeserialized = SaplingNotePlaintext::attempt_sapling_enc_decryption_deserialization(output.encCiphertext, output.ephemeralKey, outPt->esk, outPt->pk_d); + + if (!optDeserialized) { + // The transaction would not have entered the wallet unless + // its plaintest had been succesfully decrypted previously. + assert(false); + } + + auto maybe_pt = SaplingNotePlaintext::plaintext_checks_without_height( + *optDeserialized, + output.ephemeralKey, + outPt->esk, + outPt->pk_d, + output.cmu); + assert(static_cast(maybe_pt)); + auto notePt = maybe_pt.get(); + + return std::make_pair(notePt, SaplingPaymentAddress(notePt.d, outPt->pk_d)); + } + + // Couldn't recover with any of the provided OutgoingViewingKeys + return boost::none; +} + int64_t CWalletTx::GetTxTime() const { int64_t n = nTimeSmart; @@ -4982,11 +5064,17 @@ void CWallet::GetFilteredNotes( SaplingOutPoint op = pair.first; SaplingNoteData nd = pair.second; - // TODO: decide which height to use here instead of wtx.nExpiryHeight - auto maybe_pt = SaplingNotePlaintext::decrypt( - Params().GetConsensus(), - wtx.nExpiryHeight, - wtx.vShieldedOutput[op.n].encCiphertext, + auto optDeserialized = SaplingNotePlaintext::attempt_sapling_enc_decryption_deserialization(wtx.vShieldedOutput[op.n].encCiphertext, nd.ivk, wtx.vShieldedOutput[op.n].ephemeralKey); + + if (!optDeserialized) { + // The transaction would not have entered the wallet unless + // its plaintest had been succesfully decrypted previously. + assert(false); + } + // We don't need to check the leadbyte here: if wtx exists in + // the wallet, it must have already passed the leadbyte check + auto maybe_pt = SaplingNotePlaintext::plaintext_checks_without_height( + *optDeserialized, nd.ivk, wtx.vShieldedOutput[op.n].ephemeralKey, wtx.vShieldedOutput[op.n].cmu); diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index d0696afd6..4d6bcb95a 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -567,10 +567,16 @@ public: boost::optional> DecryptSaplingNote(const Consensus::Params& params, int height, SaplingOutPoint op) const; + boost::optional> DecryptSaplingNoteWithoutLeadByteCheck(SaplingOutPoint op) const; boost::optional> RecoverSaplingNote(const Consensus::Params& params, int height, SaplingOutPoint op, std::set& ovks) const; + boost::optional> RecoverSaplingNoteWithoutLeadByteCheck(SaplingOutPoint op, std::set& ovks) const; //! filter decides which addresses will count towards the debit CAmount GetDebit(const isminefilter& filter) const; @@ -1221,7 +1227,7 @@ public: const uint256& hSig, uint8_t n) const; mapSproutNoteData_t FindMySproutNotes(const CTransaction& tx) const; - std::pair FindMySaplingNotes(const CTransaction& tx) const; + std::pair FindMySaplingNotes(const CTransaction& tx, int height) const; bool IsSproutNullifierFromMe(const uint256& nullifier) const; bool IsSaplingNullifierFromMe(const uint256& nullifier) const; diff --git a/src/zcash/Note.cpp b/src/zcash/Note.cpp index 76fcb1be2..08da32ed6 100644 --- a/src/zcash/Note.cpp +++ b/src/zcash/Note.cpp @@ -209,34 +209,40 @@ boost::optional SaplingNotePlaintext::decrypt( const uint256 &cmu ) { - auto pt = AttemptSaplingEncDecryption(ciphertext, ivk, epk); - if (!pt) { - return boost::none; - } - - // Deserialize from the plaintext - SaplingNotePlaintext ret; - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << pt.get(); - ss >> ret; - assert(ss.size() == 0); - - // Check leadbyte is allowed at block height - if (!plaintext_version_is_valid(params, height, ret.leadByte)) { + auto ret = attempt_sapling_enc_decryption_deserialization(ciphertext, ivk, epk); + + if (!ret) { return boost::none; + } else { + const SaplingNotePlaintext plaintext = *ret; + + // Check leadbyte is allowed at block height + if (!plaintext_version_is_valid(params, height, plaintext.leadByte)) { + return boost::none; + } + + return plaintext_checks_without_height(plaintext, ivk, epk, cmu); } +} +boost::optional SaplingNotePlaintext::plaintext_checks_without_height( + const SaplingNotePlaintext &plaintext, + const uint256 &ivk, + const uint256 &epk, + const uint256 &cmu +) +{ uint256 pk_d; - if (!librustzcash_ivk_to_pkd(ivk.begin(), ret.d.data(), pk_d.begin())) { + if (!librustzcash_ivk_to_pkd(ivk.begin(), plaintext.d.data(), pk_d.begin())) { return boost::none; } uint256 cmu_expected; - uint256 rcm = ret.rcm(); + uint256 rcm = plaintext.rcm(); if (!librustzcash_sapling_compute_cm( - ret.d.data(), + plaintext.d.data(), pk_d.begin(), - ret.value(), + plaintext.value(), rcm.begin(), cmu_expected.begin() )) @@ -248,12 +254,12 @@ boost::optional SaplingNotePlaintext::decrypt( return boost::none; } - if (ret.leadByte == 0x02) { + if (plaintext.leadByte == 0x02) { // ZIP 212: Check that epk is consistent to prevent against linkability // attacks without relying on the soundness of the SNARK. uint256 expected_epk; - uint256 esk = ret.generate_esk(); - if (!librustzcash_sapling_ka_derivepublic(ret.d.data(), esk.begin(), expected_epk.begin())) { + uint256 esk = plaintext.generate_esk(); + if (!librustzcash_sapling_ka_derivepublic(plaintext.d.data(), esk.begin(), expected_epk.begin())) { return boost::none; } if (expected_epk != epk) { @@ -261,7 +267,29 @@ boost::optional SaplingNotePlaintext::decrypt( } } - return ret; + return plaintext; +} + +boost::optional SaplingNotePlaintext::attempt_sapling_enc_decryption_deserialization( + const SaplingEncCiphertext &ciphertext, + const uint256 &ivk, + const uint256 &epk +) +{ + auto encPlaintext = AttemptSaplingEncDecryption(ciphertext, ivk, epk); + + if (!encPlaintext) { + return boost::none; + }; + + // Deserialize from the plaintext + SaplingNotePlaintext plaintext; + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << encPlaintext.get(); + ss >> plaintext; + assert(ss.size() == 0); + + return plaintext; } boost::optional SaplingNotePlaintext::decrypt( @@ -274,26 +302,33 @@ boost::optional SaplingNotePlaintext::decrypt( const uint256 &cmu ) { - auto pt = AttemptSaplingEncDecryption(ciphertext, epk, esk, pk_d); - if (!pt) { - return boost::none; - } - - // Deserialize from the plaintext - SaplingNotePlaintext ret; - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << pt.get(); - ss >> ret; - assert(ss.size() == 0); - - // Check leadbyte is legible at block height - if (!plaintext_version_is_valid(params, height, ret.leadByte)) { + auto ret = attempt_sapling_enc_decryption_deserialization(ciphertext, epk, esk, pk_d); + + if (!ret) { return boost::none; + } else { + SaplingNotePlaintext plaintext = *ret; + + // Check leadbyte is allowed at block height + if (!plaintext_version_is_valid(params, height, plaintext.leadByte)) { + return boost::none; + } + + return plaintext_checks_without_height(plaintext, epk, esk, pk_d, cmu); } +} +boost::optional SaplingNotePlaintext::plaintext_checks_without_height( + const SaplingNotePlaintext &plaintext, + const uint256 &epk, + const uint256 &esk, + const uint256 &pk_d, + const uint256 &cmu +) +{ // Check that epk is consistent with esk uint256 expected_epk; - if (!librustzcash_sapling_ka_derivepublic(ret.d.data(), esk.begin(), expected_epk.begin())) { + if (!librustzcash_sapling_ka_derivepublic(plaintext.d.data(), esk.begin(), expected_epk.begin())) { return boost::none; } if (expected_epk != epk) { @@ -301,11 +336,11 @@ boost::optional SaplingNotePlaintext::decrypt( } uint256 cmu_expected; - uint256 rcm = ret.rcm(); + uint256 rcm = plaintext.rcm(); if (!librustzcash_sapling_compute_cm( - ret.d.data(), + plaintext.d.data(), pk_d.begin(), - ret.value(), + plaintext.value(), rcm.begin(), cmu_expected.begin() )) @@ -317,15 +352,38 @@ boost::optional SaplingNotePlaintext::decrypt( return boost::none; } - if (ret.leadByte == 0x02) { + if (plaintext.leadByte == 0x02) { // ZIP 212: Additionally check that the esk provided to this function // is consistent with the esk we can derive - if (esk != ret.generate_esk()) { + if (esk != plaintext.generate_esk()) { return boost::none; } } - return ret; + return plaintext; +} + +boost::optional SaplingNotePlaintext::attempt_sapling_enc_decryption_deserialization( + const SaplingEncCiphertext &ciphertext, + const uint256 &epk, + const uint256 &esk, + const uint256 &pk_d +) +{ + auto encPlaintext = AttemptSaplingEncDecryption(ciphertext, epk, esk, pk_d); + + if (!encPlaintext) { + return boost::none; + }; + + // Deserialize from the plaintext + SaplingNotePlaintext plaintext; + CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); + ss << encPlaintext.get(); + ss >> plaintext; + assert(ss.size() == 0); + + return plaintext; } boost::optional SaplingNotePlaintext::encrypt(const uint256& pk_d) const diff --git a/src/zcash/Note.hpp b/src/zcash/Note.hpp index 134c289eb..b9df41547 100644 --- a/src/zcash/Note.hpp +++ b/src/zcash/Note.hpp @@ -167,6 +167,19 @@ public: const uint256 &cmu ); + static boost::optional plaintext_checks_without_height( + const SaplingNotePlaintext &plaintext, + const uint256 &ivk, + const uint256 &epk, + const uint256 &cmu + ); + + static boost::optional attempt_sapling_enc_decryption_deserialization( + const SaplingEncCiphertext &ciphertext, + const uint256 &ivk, + const uint256 &epk + ); + static boost::optional decrypt( const Consensus::Params& params, int height, @@ -177,6 +190,21 @@ public: const uint256 &cmu ); + static boost::optional plaintext_checks_without_height( + const SaplingNotePlaintext &plaintext, + const uint256 &epk, + const uint256 &esk, + const uint256 &pk_d, + const uint256 &cmu + ); + + static boost::optional attempt_sapling_enc_decryption_deserialization( + const SaplingEncCiphertext &ciphertext, + const uint256 &epk, + const uint256 &esk, + const uint256 &pk_d + ); + boost::optional note(const SaplingIncomingViewingKey& ivk) const; virtual ~SaplingNotePlaintext() {} diff --git a/src/zcbenchmarks.cpp b/src/zcbenchmarks.cpp index 33378f2db..4311f7bf7 100644 --- a/src/zcbenchmarks.cpp +++ b/src/zcbenchmarks.cpp @@ -307,7 +307,7 @@ double benchmark_try_decrypt_sapling_notes(size_t nKeys) struct timeval tv_start; timer_start(tv_start); - auto noteDataMapAndAddressesToAdd = wallet.FindMySaplingNotes(tx); + auto noteDataMapAndAddressesToAdd = wallet.FindMySaplingNotes(tx, 1); assert(noteDataMapAndAddressesToAdd.first.empty()); return timer_stop(tv_start); } From 7a1d1191707c44cc6addfb47d4678d796a357f8a Mon Sep 17 00:00:00 2001 From: therealyingtong Date: Sat, 20 Jun 2020 00:28:06 +0800 Subject: [PATCH 091/725] Add gtests for v2 plaintexts --- src/gtest/test_checktransaction.cpp | 77 ++++ src/gtest/test_noteencryption.cpp | 535 ++++++++++++++++++------ src/gtest/test_transaction_builder.cpp | 121 ++++++ src/wallet/gtest/test_wallet.cpp | 557 +++++++++++++------------ 4 files changed, 905 insertions(+), 385 deletions(-) diff --git a/src/gtest/test_checktransaction.cpp b/src/gtest/test_checktransaction.cpp index 233dcb03c..0879bfb01 100644 --- a/src/gtest/test_checktransaction.cpp +++ b/src/gtest/test_checktransaction.cpp @@ -1284,3 +1284,80 @@ TEST(CheckTransaction, HeartwoodEnforcesSaplingRulesOnShieldedCoinbase) { RegtestDeactivateHeartwood(); } + +// Check that the consensus rules relevant to valueBalance, vShieldedOutput, and +// bindingSig from https://zips.z.cash/protocol/protocol.pdf#txnencoding are +// applied to coinbase transactions. +TEST(CheckTransaction, CanopyEnforcesSaplingRulesOnShieldedCoinbase) { + RegtestActivateCanopy(false, Consensus::NetworkUpgrade::ALWAYS_ACTIVE); + auto chainparams = Params(); + + uint256 ovk; + auto note = libzcash::SaplingNote( + libzcash::SaplingSpendingKey::random().default_address(), CAmount(123456), 0x02); + auto output = OutputDescriptionInfo(ovk, note, {{0xF6}}); + + CMutableTransaction mtx = GetValidTransaction(); + mtx.fOverwintered = true; + mtx.nVersionGroupId = SAPLING_VERSION_GROUP_ID; + mtx.nVersion = SAPLING_TX_VERSION; + + mtx.vin.resize(1); + mtx.vin[0].prevout.SetNull(); + mtx.vin[0].scriptSig << 123; + mtx.vJoinSplit.resize(0); + mtx.valueBalance = -1000; + + // Coinbase transaction should fail non-contextual checks with no shielded + // outputs and non-zero valueBalance. + { + CTransaction tx(mtx); + EXPECT_TRUE(tx.IsCoinBase()); + + MockCValidationState state; + EXPECT_CALL(state, DoS(100, false, REJECT_INVALID, "bad-txns-valuebalance-nonzero", false)).Times(1); + EXPECT_FALSE(CheckTransactionWithoutProofVerification(tx, state)); + } + + // Add a Sapling output. + auto ctx = librustzcash_sapling_proving_ctx_init(); + auto odesc = output.Build(ctx).get(); + librustzcash_sapling_proving_ctx_free(ctx); + mtx.vShieldedOutput.push_back(odesc); + + // Coinbase transaction should fail non-contextual checks with valueBalance + // out of range. + { + mtx.valueBalance = MAX_MONEY + 1; + CTransaction tx(mtx); + EXPECT_TRUE(tx.IsCoinBase()); + + MockCValidationState state; + EXPECT_CALL(state, DoS(100, false, REJECT_INVALID, "bad-txns-valuebalance-toolarge", false)).Times(1); + EXPECT_FALSE(CheckTransactionWithoutProofVerification(tx, state)); + } + { + mtx.valueBalance = -MAX_MONEY - 1; + CTransaction tx(mtx); + EXPECT_TRUE(tx.IsCoinBase()); + + MockCValidationState state; + EXPECT_CALL(state, DoS(100, false, REJECT_INVALID, "bad-txns-valuebalance-toolarge", false)).Times(1); + EXPECT_FALSE(CheckTransactionWithoutProofVerification(tx, state)); + } + + mtx.valueBalance = -1000; + CTransaction tx(mtx); + EXPECT_TRUE(tx.IsCoinBase()); + + // Coinbase transaction should now pass non-contextual checks. + MockCValidationState state; + EXPECT_TRUE(CheckTransactionWithoutProofVerification(tx, state)); + + // Coinbase transaction does not pass contextual checks, as bindingSig + // consensus rule is enforced. + EXPECT_CALL(state, DoS(100, false, REJECT_INVALID, "bad-txns-sapling-binding-signature-invalid", false)).Times(1); + ContextualCheckTransaction(tx, state, chainparams, 10, 57); + + RegtestDeactivateCanopy(); +} diff --git a/src/gtest/test_noteencryption.cpp b/src/gtest/test_noteencryption.cpp index 2d99158b2..22357d0ed 100644 --- a/src/gtest/test_noteencryption.cpp +++ b/src/gtest/test_noteencryption.cpp @@ -22,14 +22,19 @@ public: } }; -TEST(noteencryption, NotePlaintext) +TEST(NoteEncryption, NotePlaintext) { SelectParams(CBaseChainParams::REGTEST); - const Consensus::Params& params = Params().GetConsensus(); int overwinterActivationHeight = 5; int saplingActivationHeight = 30; + int canopyActivationHeight = 70; UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, overwinterActivationHeight); UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, saplingActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_CANOPY, canopyActivationHeight); + auto params = Params().GetConsensus(); + + unsigned char leadBytes[] = {0x01, 0x02}; + int decryptionHeights[] = {saplingActivationHeight, canopyActivationHeight}; using namespace libzcash; auto xsk = SaplingSpendingKey(uint256()).expanded_spending_key(); @@ -43,139 +48,417 @@ TEST(noteencryption, NotePlaintext) memo[i] = (unsigned char) i; } - SaplingNote note(addr, 39393, 0x01); - auto cmu_opt = note.cmu(); - if (!cmu_opt) { - FAIL(); - } - uint256 cmu = cmu_opt.get(); - SaplingNotePlaintext pt(note, memo); + for (int ver = 0; ver < sizeof(leadBytes); ver++){ + SaplingNote note(addr, 39393, leadBytes[ver]); + auto cmu_opt = note.cmu(); + if (!cmu_opt) { + FAIL(); + } + uint256 cmu = cmu_opt.get(); + SaplingNotePlaintext pt(note, memo); - auto res = pt.encrypt(addr.pk_d); - if (!res) { - FAIL(); - } + auto res = pt.encrypt(addr.pk_d); + if (!res) { + FAIL(); + } - auto enc = res.get(); + auto enc = res.get(); - auto ct = enc.first; - auto encryptor = enc.second; - auto epk = encryptor.get_epk(); + auto ct = enc.first; + auto encryptor = enc.second; + auto epk = encryptor.get_epk(); - // Try to decrypt with incorrect commitment - ASSERT_FALSE(SaplingNotePlaintext::decrypt( - params, - saplingActivationHeight, - ct, - ivk, - epk, - uint256() - )); - - // Try to decrypt with correct commitment - auto foo = SaplingNotePlaintext::decrypt( - params, - saplingActivationHeight, - ct, - ivk, - epk, - cmu - ); - - if (!foo) { - FAIL(); - } - - auto bar = foo.get(); - - ASSERT_TRUE(bar.value() == pt.value()); - ASSERT_TRUE(bar.memo() == pt.memo()); - ASSERT_TRUE(bar.d == pt.d); - ASSERT_TRUE(bar.rcm() == pt.rcm()); - - auto foobar = bar.note(ivk); - - if (!foobar) { - FAIL(); - } - - auto new_note = foobar.get(); - - ASSERT_TRUE(note.value() == new_note.value()); - ASSERT_TRUE(note.d == new_note.d); - ASSERT_TRUE(note.pk_d == new_note.pk_d); - ASSERT_TRUE(note.rcm() == new_note.rcm()); - ASSERT_TRUE(note.cmu() == new_note.cmu()); - - SaplingOutgoingPlaintext out_pt; - out_pt.pk_d = note.pk_d; - out_pt.esk = encryptor.get_esk(); - - auto ovk = random_uint256(); - auto cv = random_uint256(); - auto cm = random_uint256(); - - auto out_ct = out_pt.encrypt( - ovk, - cv, - cm, - encryptor - ); - - auto decrypted_out_ct = out_pt.decrypt( - out_ct, - ovk, - cv, - cm, - encryptor.get_epk() - ); - - if (!decrypted_out_ct) { - FAIL(); - } - - auto decrypted_out_ct_unwrapped = decrypted_out_ct.get(); - - ASSERT_TRUE(decrypted_out_ct_unwrapped.pk_d == out_pt.pk_d); - ASSERT_TRUE(decrypted_out_ct_unwrapped.esk == out_pt.esk); - - // Test sender won't accept invalid commitments - ASSERT_FALSE( - SaplingNotePlaintext::decrypt( + // Try to decrypt with incorrect commitment + ASSERT_FALSE(SaplingNotePlaintext::decrypt( params, - saplingActivationHeight, + decryptionHeights[ver], + ct, + ivk, + epk, + uint256() + )); + + // Try to decrypt with correct commitment + auto foo = SaplingNotePlaintext::decrypt( + params, + decryptionHeights[ver], + ct, + ivk, + epk, + cmu + ); + + if (!foo) { + FAIL(); + } + + auto bar = foo.get(); + + ASSERT_TRUE(bar.value() == pt.value()); + ASSERT_TRUE(bar.memo() == pt.memo()); + ASSERT_TRUE(bar.d == pt.d); + ASSERT_TRUE(bar.rcm() == pt.rcm()); + + auto foobar = bar.note(ivk); + + if (!foobar) { + FAIL(); + } + + auto new_note = foobar.get(); + + ASSERT_TRUE(note.value() == new_note.value()); + ASSERT_TRUE(note.d == new_note.d); + ASSERT_TRUE(note.pk_d == new_note.pk_d); + ASSERT_TRUE(note.rcm() == new_note.rcm()); + ASSERT_TRUE(note.cmu() == new_note.cmu()); + + SaplingOutgoingPlaintext out_pt; + out_pt.pk_d = note.pk_d; + out_pt.esk = encryptor.get_esk(); + + auto ovk = random_uint256(); + auto cv = random_uint256(); + auto cm = random_uint256(); + + auto out_ct = out_pt.encrypt( + ovk, + cv, + cm, + encryptor + ); + + auto decrypted_out_ct = out_pt.decrypt( + out_ct, + ovk, + cv, + cm, + encryptor.get_epk() + ); + + if (!decrypted_out_ct) { + FAIL(); + } + + auto decrypted_out_ct_unwrapped = decrypted_out_ct.get(); + + ASSERT_TRUE(decrypted_out_ct_unwrapped.pk_d == out_pt.pk_d); + ASSERT_TRUE(decrypted_out_ct_unwrapped.esk == out_pt.esk); + + // Test sender won't accept invalid commitments + ASSERT_FALSE( + SaplingNotePlaintext::decrypt( + params, + decryptionHeights[ver], + ct, + epk, + decrypted_out_ct_unwrapped.esk, + decrypted_out_ct_unwrapped.pk_d, + uint256() + ) + ); + + // Test sender can decrypt the note ciphertext. + foo = SaplingNotePlaintext::decrypt( + params, + decryptionHeights[ver], ct, epk, decrypted_out_ct_unwrapped.esk, decrypted_out_ct_unwrapped.pk_d, - uint256() - ) - ); + cmu + ); - // Test sender can decrypt the note ciphertext. - foo = SaplingNotePlaintext::decrypt( - params, - saplingActivationHeight, - ct, - epk, - decrypted_out_ct_unwrapped.esk, - decrypted_out_ct_unwrapped.pk_d, - cmu - ); + if (!foo) { + FAIL(); + } - if (!foo) { - FAIL(); + bar = foo.get(); + + ASSERT_TRUE(bar.value() == pt.value()); + ASSERT_TRUE(bar.memo() == pt.memo()); + ASSERT_TRUE(bar.d == pt.d); + ASSERT_TRUE(bar.rcm() == pt.rcm()); } - bar = foo.get(); - - ASSERT_TRUE(bar.value() == pt.value()); - ASSERT_TRUE(bar.memo() == pt.memo()); - ASSERT_TRUE(bar.d == pt.d); - ASSERT_TRUE(bar.rcm() == pt.rcm()); + // Revert to test default + RegtestDeactivateCanopy(); + RegtestDeactivateHeartwood(); + RegtestDeactivateSapling(); } -TEST(noteencryption, SaplingApi) +TEST(NoteEncryption, RejectsInvalidNotePlaintextVersion) +{ + SelectParams(CBaseChainParams::REGTEST); + int overwinterActivationHeight = 5; + int saplingActivationHeight = 30; + int canopyActivationHeight = 70; + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, overwinterActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, saplingActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_CANOPY, canopyActivationHeight); + auto params = Params().GetConsensus(); + + using namespace libzcash; + auto xsk = SaplingSpendingKey(uint256()).expanded_spending_key(); + auto fvk = xsk.full_viewing_key(); + auto ivk = fvk.in_viewing_key(); + SaplingPaymentAddress addr = *ivk.address({0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + + std::array memo; + for (size_t i = 0; i < ZC_MEMO_SIZE; i++) { + // Fill the message with dummy data + memo[i] = (unsigned char) i; + } + + { + // non-0x01 received before Canopy activation height + SaplingNote note(addr, 39393, 0x02); + auto cmu_opt = note.cmu(); + if (!cmu_opt) { + FAIL(); + } + uint256 cmu = cmu_opt.get(); + SaplingNotePlaintext pt(note, memo); + + auto res = pt.encrypt(addr.pk_d); + if (!res) { + FAIL(); + } + + auto enc = res.get(); + + auto ct = enc.first; + auto encryptor = enc.second; + auto epk = encryptor.get_epk(); + + ASSERT_FALSE(SaplingNotePlaintext::decrypt( + params, + canopyActivationHeight - 1, + ct, + ivk, + epk, + cmu + )); + } + + { + // non-{0x01,0x02} received after Canopy activation and before grace period has elapsed + SaplingNote note(addr, 39393, 0x03); + int height1 = canopyActivationHeight; + int height2 = canopyActivationHeight + (ZIP212_GRACE_PERIOD) - 1; + int heights[] = {height1, height2}; + + for (int j = 0; j < sizeof(heights) / sizeof(int); j++) { + auto cmu_opt = note.cmu(); + if (!cmu_opt) { + FAIL(); + } + uint256 cmu = cmu_opt.get(); + SaplingNotePlaintext pt(note, memo); + + auto res = pt.encrypt(addr.pk_d); + if (!res) { + FAIL(); + } + + auto enc = res.get(); + + auto ct = enc.first; + auto encryptor = enc.second; + auto epk = encryptor.get_epk(); + + ASSERT_FALSE(SaplingNotePlaintext::decrypt( + params, + heights[j], + ct, + ivk, + epk, + cmu + )); + } + } + + { + // non-0x02 received past (Canopy activation height + grace period) + SaplingNote note(addr, 39393, 0x01); + auto cmu_opt = note.cmu(); + if (!cmu_opt) { + FAIL(); + } + uint256 cmu = cmu_opt.get(); + SaplingNotePlaintext pt(note, memo); + + auto res = pt.encrypt(addr.pk_d); + if (!res) { + FAIL(); + } + + auto enc = res.get(); + + auto ct = enc.first; + auto encryptor = enc.second; + auto epk = encryptor.get_epk(); + + ASSERT_FALSE(SaplingNotePlaintext::decrypt( + params, + canopyActivationHeight + ZIP212_GRACE_PERIOD, + ct, + ivk, + epk, + cmu + )); + } + + // Revert to test default + RegtestDeactivateCanopy(); + RegtestDeactivateHeartwood(); + RegtestDeactivateSapling(); +} + +TEST(NoteEncryption, AcceptsValidNotePlaintextVersion) +{ + SelectParams(CBaseChainParams::REGTEST); + int overwinterActivationHeight = 5; + int saplingActivationHeight = 30; + int canopyActivationHeight = 70; + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, overwinterActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, saplingActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_CANOPY, canopyActivationHeight); + auto params = Params().GetConsensus(); + + using namespace libzcash; + auto xsk = SaplingSpendingKey(uint256()).expanded_spending_key(); + auto fvk = xsk.full_viewing_key(); + auto ivk = fvk.in_viewing_key(); + SaplingPaymentAddress addr = *ivk.address({0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}); + + std::array memo; + for (size_t i = 0; i < ZC_MEMO_SIZE; i++) { + // Fill the message with dummy data + memo[i] = (unsigned char) i; + } + + { + // 0x01 received before Canopy activation height + SaplingNote note(addr, 39393, 0x01); + auto cmu_opt = note.cmu(); + if (!cmu_opt) { + FAIL(); + } + uint256 cmu = cmu_opt.get(); + SaplingNotePlaintext pt(note, memo); + + auto res = pt.encrypt(addr.pk_d); + if (!res) { + FAIL(); + } + + auto enc = res.get(); + + auto ct = enc.first; + auto encryptor = enc.second; + auto epk = encryptor.get_epk(); + + auto plaintext = SaplingNotePlaintext::decrypt( + params, + canopyActivationHeight - 1, + ct, + ivk, + epk, + cmu + ); + + if (!plaintext) { + FAIL(); + } + } + + { + // {0x01,0x02} received after Canopy activation and before grace period has elapsed + unsigned char leadBytes[] = {0x01, 0x02}; + int height1 = canopyActivationHeight; + int height2 = canopyActivationHeight + (ZIP212_GRACE_PERIOD) - 1; + int heights[] = {height1, height2}; + + for (int i = 0; i < sizeof(leadBytes); i++) { + for (int j = 0; j < sizeof(heights) / sizeof(int); j++) { + SaplingNote note(addr, 39393, leadBytes[i]); + auto cmu_opt = note.cmu(); + if (!cmu_opt) { + FAIL(); + } + uint256 cmu = cmu_opt.get(); + SaplingNotePlaintext pt(note, memo); + + auto res = pt.encrypt(addr.pk_d); + if (!res) { + FAIL(); + } + + auto enc = res.get(); + + auto ct = enc.first; + auto encryptor = enc.second; + auto epk = encryptor.get_epk(); + + auto plaintext = SaplingNotePlaintext::decrypt( + params, + heights[j], + ct, + ivk, + epk, + cmu + ); + + if (!plaintext) { + FAIL(); + } + } + } + } + + { + // 0x02 received past (Canopy activation height + grace period) + SaplingNote note(addr, 39393, 0x02); + auto cmu_opt = note.cmu(); + if (!cmu_opt) { + FAIL(); + } + uint256 cmu = cmu_opt.get(); + SaplingNotePlaintext pt(note, memo); + + auto res = pt.encrypt(addr.pk_d); + if (!res) { + FAIL(); + } + + auto enc = res.get(); + + auto ct = enc.first; + auto encryptor = enc.second; + auto epk = encryptor.get_epk(); + + auto plaintext = SaplingNotePlaintext::decrypt( + params, + canopyActivationHeight + ZIP212_GRACE_PERIOD, + ct, + ivk, + epk, + cmu + ); + + if (!plaintext) { + FAIL(); + } + } + + // Revert to test default + RegtestDeactivateCanopy(); + RegtestDeactivateHeartwood(); + RegtestDeactivateSapling(); +} + +TEST(NoteEncryption, SaplingApi) { using namespace libzcash; @@ -358,7 +641,7 @@ TEST(noteencryption, SaplingApi) )); } -TEST(noteencryption, api) +TEST(NoteEncryption, api) { uint256 sk_enc = ZCNoteEncryption::generate_privkey(uint252(uint256S("21035d60bc1983e37950ce4803418a8fb33ea68d5b937ca382ecbae7564d6a07"))); uint256 pk_enc = ZCNoteEncryption::generate_pubkey(sk_enc); @@ -463,7 +746,7 @@ uint256 test_prf( return ret; } -TEST(noteencryption, PrfAddr) +TEST(NoteEncryption, PrfAddr) { for (size_t i = 0; i < 100; i++) { uint252 a_sk = libzcash::random_uint252(); @@ -483,7 +766,7 @@ TEST(noteencryption, PrfAddr) } } -TEST(noteencryption, PrfNf) +TEST(NoteEncryption, PrfNf) { for (size_t i = 0; i < 100; i++) { uint252 a_sk = libzcash::random_uint252(); @@ -494,7 +777,7 @@ TEST(noteencryption, PrfNf) } } -TEST(noteencryption, PrfPk) +TEST(NoteEncryption, PrfPk) { for (size_t i = 0; i < 100; i++) { uint252 a_sk = libzcash::random_uint252(); @@ -517,7 +800,7 @@ TEST(noteencryption, PrfPk) ASSERT_THROW(PRF_pk(dummy_a, 2, dummy_b), std::domain_error); } -TEST(noteencryption, PrfRho) +TEST(NoteEncryption, PrfRho) { for (size_t i = 0; i < 100; i++) { uint252 phi = libzcash::random_uint252(); @@ -540,7 +823,7 @@ TEST(noteencryption, PrfRho) ASSERT_THROW(PRF_rho(dummy_a, 2, dummy_b), std::domain_error); } -TEST(noteencryption, uint252) +TEST(NoteEncryption, uint252) { ASSERT_THROW(uint252(uint256S("f6da8716682d600f74fc16bd0187faad6a26b4aa4c24d5c055b216d94516847e")), std::domain_error); } diff --git a/src/gtest/test_transaction_builder.cpp b/src/gtest/test_transaction_builder.cpp index 3cc382265..633b83915 100644 --- a/src/gtest/test_transaction_builder.cpp +++ b/src/gtest/test_transaction_builder.cpp @@ -1,5 +1,6 @@ #include "chainparams.h" #include "consensus/params.h" +#include "consensus/consensus.h" #include "consensus/validation.h" #include "key_io.h" #include "main.h" @@ -495,3 +496,123 @@ TEST(TransactionBuilder, CheckSaplingTxVersion) // Revert to default UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT); } + +TEST(TransactionBuilder, RejectsInvalidNotePlaintextVersion) +{ + SelectParams(CBaseChainParams::REGTEST); + int overwinterActivationHeight = 5; + int saplingActivationHeight = 30; + int canopyActivationHeight = 70; + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, overwinterActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, saplingActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_CANOPY, canopyActivationHeight); + auto consensusParams = Params().GetConsensus(); + + auto sk = libzcash::SaplingSpendingKey::random(); + auto expsk = sk.expanded_spending_key(); + auto pk = sk.default_address(); + + SaplingMerkleTree tree; + + { + // non-0x01 received before Canopy activation height + auto builder = TransactionBuilder(consensusParams, canopyActivationHeight - 1); + libzcash::SaplingNote note(pk, 50000, 0x02); + try { + builder.AddSaplingSpend(expsk, note, uint256(), tree.witness()); + } catch (std::runtime_error const & err) { + EXPECT_EQ(err.what(), std::string("TransactionBuilder: invalid note plaintext version")); + } catch(...) { + FAIL() << "Expected std::runtime_error"; + } + } + + { + // non-{0x01,0x02} received after Canopy activation and before grace period has elapsed + libzcash::SaplingNote note(pk, 50000, 0x03); + int height1 = canopyActivationHeight - 1; + int height2 = canopyActivationHeight + (ZIP212_GRACE_PERIOD) - 2; + int heights[] = {height1, height2}; + + for (int j = 0; j < sizeof(heights) / sizeof(int); j++) { + auto builder = TransactionBuilder(consensusParams, heights[j]); + try { + builder.AddSaplingSpend(expsk, note, uint256(), tree.witness()); + } catch (std::runtime_error const & err) { + EXPECT_EQ(err.what(), std::string("TransactionBuilder: invalid note plaintext version")); + } catch(...) { + FAIL() << "Expected std::runtime_error"; + } + } + } + + { + // non-0x02 received past (Canopy activation height + grace period) + auto builder = TransactionBuilder(consensusParams, canopyActivationHeight + ZIP212_GRACE_PERIOD); + libzcash::SaplingNote note(pk, 50000, 0x01); + try { + builder.AddSaplingSpend(expsk, note, uint256(), tree.witness()); + } catch (std::runtime_error const & err) { + EXPECT_EQ(err.what(), std::string("TransactionBuilder: invalid note plaintext version")); + } catch(...) { + FAIL() << "Expected std::runtime_error"; + } + } + + // Revert to default + RegtestDeactivateCanopy(); + RegtestDeactivateSapling(); +} + +TEST(TransactionBuilder, AcceptsValidNotePlaintextVersion) +{ + SelectParams(CBaseChainParams::REGTEST); + int overwinterActivationHeight = 5; + int saplingActivationHeight = 30; + int canopyActivationHeight = 70; + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, overwinterActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, saplingActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_CANOPY, canopyActivationHeight); + auto consensusParams = Params().GetConsensus(); + + auto sk = libzcash::SaplingSpendingKey::random(); + auto expsk = sk.expanded_spending_key(); + auto pk = sk.default_address(); + + SaplingMerkleTree tree; + + { + // 0x01 received before Canopy activation height + auto builder = TransactionBuilder(consensusParams, canopyActivationHeight - 1); + libzcash::SaplingNote note(pk, 50000, 0x01); + ASSERT_NO_THROW(builder.AddSaplingSpend(expsk, note, uint256(), tree.witness())); + } + + { + // {0x01,0x02} received after Canopy activation and before grace period has elapsed + unsigned char leadBytes[] = {0x01, 0x02}; + int height1 = canopyActivationHeight - 1; + int height2 = canopyActivationHeight + (ZIP212_GRACE_PERIOD) - 2; + int heights[] = {height1, height2}; + + for (int i = 0; i < sizeof(leadBytes); i++) { + for (int j = 0; j < sizeof(heights) / sizeof(int); j++) { + printf("height %d: %d\n", j, heights[j]); + auto builder = TransactionBuilder(consensusParams, heights[j]); + libzcash::SaplingNote note(pk, 50000, leadBytes[i]); + ASSERT_NO_THROW(builder.AddSaplingSpend(expsk, note, uint256(), tree.witness())); + } + } + } + + { + // 0x02 received past (Canopy activation height + grace period) + auto builder = TransactionBuilder(consensusParams, canopyActivationHeight + ZIP212_GRACE_PERIOD - 1); + libzcash::SaplingNote note(pk, 50000, 0x02); + ASSERT_NO_THROW(builder.AddSaplingSpend(expsk, note, uint256(), tree.witness())); + } + + // Revert to default + RegtestDeactivateCanopy(); + RegtestDeactivateSapling(); +} diff --git a/src/wallet/gtest/test_wallet.cpp b/src/wallet/gtest/test_wallet.cpp index 348b2e022..bb29793cb 100644 --- a/src/wallet/gtest/test_wallet.cpp +++ b/src/wallet/gtest/test_wallet.cpp @@ -376,57 +376,70 @@ TEST(WalletTests, SetSproutNoteAddrsInCWalletTx) { } TEST(WalletTests, SetSaplingNoteAddrsInCWalletTx) { - auto consensusParams = RegtestActivateSapling(); + SelectParams(CBaseChainParams::REGTEST); + int overwinterActivationHeight = 5; + int saplingActivationHeight = 30; + int canopyActivationHeight = 70; + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, overwinterActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, saplingActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_CANOPY, canopyActivationHeight); + auto consensusParams = Params().GetConsensus(); - TestWallet wallet; - LOCK(wallet.cs_wallet); + unsigned char leadBytes[] = {0x01, 0x02}; + int builderHeights[] = {saplingActivationHeight, canopyActivationHeight}; - auto sk = GetTestMasterSaplingSpendingKey(); - auto expsk = sk.expsk; - auto fvk = expsk.full_viewing_key(); - auto ivk = fvk.in_viewing_key(); - auto pk = sk.DefaultAddress(); + for (int ver = 0; ver < sizeof(leadBytes); ver++) { + TestWallet wallet; + LOCK(wallet.cs_wallet); - libzcash::SaplingNote note(pk, 50000, 0x01); - auto cm = note.cmu().get(); - SaplingMerkleTree tree; - tree.append(cm); - auto anchor = tree.root(); - auto witness = tree.witness(); + auto sk = GetTestMasterSaplingSpendingKey(); + auto expsk = sk.expsk; + auto fvk = expsk.full_viewing_key(); + auto ivk = fvk.in_viewing_key(); + auto pk = sk.DefaultAddress(); - auto nf = note.nullifier(fvk, witness.position()); - ASSERT_TRUE(nf); - uint256 nullifier = nf.get(); + libzcash::SaplingNote note(pk, 50000, leadBytes[ver]); + auto cm = note.cmu().get(); + SaplingMerkleTree tree; + tree.append(cm); + auto anchor = tree.root(); + auto witness = tree.witness(); - auto builder = TransactionBuilder(consensusParams, 1); - builder.AddSaplingSpend(expsk, note, anchor, witness); - builder.AddSaplingOutput(fvk.ovk, pk, 50000, {}); - builder.SetFee(0); - auto tx = builder.Build().GetTxOrThrow(); + auto nf = note.nullifier(fvk, witness.position()); + ASSERT_TRUE(nf); + uint256 nullifier = nf.get(); - CWalletTx wtx {&wallet, tx}; + auto builder = TransactionBuilder(consensusParams, builderHeights[ver]); + builder.AddSaplingSpend(expsk, note, anchor, witness); + builder.AddSaplingOutput(fvk.ovk, pk, 50000, {}); + builder.SetFee(0); + auto tx = builder.Build().GetTxOrThrow(); - EXPECT_EQ(0, wtx.mapSaplingNoteData.size()); - mapSaplingNoteData_t noteData; + CWalletTx wtx {&wallet, tx}; - SaplingOutPoint op {wtx.GetHash(), 0}; - SaplingNoteData nd; - nd.nullifier = nullifier; - nd.ivk = ivk; - nd.witnesses.push_front(witness); - nd.witnessHeight = 123; - noteData.insert(std::make_pair(op, nd)); + EXPECT_EQ(0, wtx.mapSaplingNoteData.size()); + mapSaplingNoteData_t noteData; - wtx.SetSaplingNoteData(noteData); - EXPECT_EQ(noteData, wtx.mapSaplingNoteData); + SaplingOutPoint op {wtx.GetHash(), 0}; + SaplingNoteData nd; + nd.nullifier = nullifier; + nd.ivk = ivk; + nd.witnesses.push_front(witness); + nd.witnessHeight = 123; + noteData.insert(std::make_pair(op, nd)); - // Test individual fields in case equality operator is defined/changed. - EXPECT_EQ(ivk, wtx.mapSaplingNoteData[op].ivk); - EXPECT_EQ(nullifier, wtx.mapSaplingNoteData[op].nullifier); - EXPECT_EQ(nd.witnessHeight, wtx.mapSaplingNoteData[op].witnessHeight); - EXPECT_TRUE(witness == wtx.mapSaplingNoteData[op].witnesses.front()); + wtx.SetSaplingNoteData(noteData); + EXPECT_EQ(noteData, wtx.mapSaplingNoteData); + + // Test individual fields in case equality operator is defined/changed. + EXPECT_EQ(ivk, wtx.mapSaplingNoteData[op].ivk); + EXPECT_EQ(nullifier, wtx.mapSaplingNoteData[op].nullifier); + EXPECT_EQ(nd.witnessHeight, wtx.mapSaplingNoteData[op].witnessHeight); + EXPECT_TRUE(witness == wtx.mapSaplingNoteData[op].witnesses.front()); + } // Revert to default + RegtestDeactivateCanopy(); RegtestDeactivateSapling(); } @@ -534,13 +547,13 @@ TEST(WalletTests, FindMySaplingNotes) { // No Sapling notes can be found in tx which does not belong to the wallet CWalletTx wtx {&wallet, tx}; ASSERT_FALSE(wallet.HaveSaplingSpendingKey(extfvk)); - auto noteMap = wallet.FindMySaplingNotes(wtx).first; + auto noteMap = wallet.FindMySaplingNotes(wtx, 1).first; EXPECT_EQ(0, noteMap.size()); // Add spending key to wallet, so Sapling notes can be found ASSERT_TRUE(wallet.AddSaplingZKey(sk)); ASSERT_TRUE(wallet.HaveSaplingSpendingKey(extfvk)); - noteMap = wallet.FindMySaplingNotes(wtx).first; + noteMap = wallet.FindMySaplingNotes(wtx, 1).first; EXPECT_EQ(2, noteMap.size()); // Revert to default @@ -638,123 +651,136 @@ TEST(WalletTests, GetConflictedSproutNotes) { // Generate note A and spend to create note B, from which we spend to create two conflicting transactions TEST(WalletTests, GetConflictedSaplingNotes) { - auto consensusParams = RegtestActivateSapling(); + SelectParams(CBaseChainParams::REGTEST); + int overwinterActivationHeight = 5; + int saplingActivationHeight = 30; + int canopyActivationHeight = 70; + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, overwinterActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, saplingActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_CANOPY, canopyActivationHeight); + auto consensusParams = Params().GetConsensus(); - TestWallet wallet; - LOCK2(cs_main, wallet.cs_wallet); + unsigned char leadBytes[] = {0x01, 0x02}; + int builderHeights[] = {saplingActivationHeight, canopyActivationHeight}; - // Generate Sapling address - auto sk = GetTestMasterSaplingSpendingKey(); - auto expsk = sk.expsk; - auto extfvk = sk.ToXFVK(); - auto ivk = extfvk.fvk.in_viewing_key(); - auto pk = sk.DefaultAddress(); + for (int ver = 0; ver < sizeof(leadBytes); ver++) { + TestWallet wallet; + LOCK2(cs_main, wallet.cs_wallet); - ASSERT_TRUE(wallet.AddSaplingZKey(sk)); - ASSERT_TRUE(wallet.HaveSaplingSpendingKey(extfvk)); + // Generate Sapling address + auto sk = GetTestMasterSaplingSpendingKey(); + auto expsk = sk.expsk; + auto extfvk = sk.ToXFVK(); + auto ivk = extfvk.fvk.in_viewing_key(); + auto pk = sk.DefaultAddress(); - // Generate note A - libzcash::SaplingNote note(pk, 50000, 0x01); - auto cm = note.cmu().get(); - SaplingMerkleTree saplingTree; - saplingTree.append(cm); - auto anchor = saplingTree.root(); - auto witness = saplingTree.witness(); + ASSERT_TRUE(wallet.AddSaplingZKey(sk)); + ASSERT_TRUE(wallet.HaveSaplingSpendingKey(extfvk)); - // Generate tx to create output note B - auto builder = TransactionBuilder(consensusParams, 1); - builder.AddSaplingSpend(expsk, note, anchor, witness); - builder.AddSaplingOutput(extfvk.fvk.ovk, pk, 35000, {}); - auto tx = builder.Build().GetTxOrThrow(); - CWalletTx wtx {&wallet, tx}; + // Generate note A + libzcash::SaplingNote note(pk, 50000, leadBytes[ver]); + auto cm = note.cmu().get(); + SaplingMerkleTree saplingTree; + saplingTree.append(cm); + auto anchor = saplingTree.root(); + auto witness = saplingTree.witness(); - // Fake-mine the transaction - EXPECT_EQ(-1, chainActive.Height()); - SproutMerkleTree sproutTree; - CBlock block; - block.vtx.push_back(wtx); - block.hashMerkleRoot = block.BuildMerkleTree(); - auto blockHash = block.GetHash(); - CBlockIndex fakeIndex {block}; - mapBlockIndex.insert(std::make_pair(blockHash, &fakeIndex)); - chainActive.SetTip(&fakeIndex); - EXPECT_TRUE(chainActive.Contains(&fakeIndex)); - EXPECT_EQ(0, chainActive.Height()); + // Generate tx to create output note B + auto builder = TransactionBuilder(consensusParams, builderHeights[ver]); + builder.AddSaplingSpend(expsk, note, anchor, witness); + builder.AddSaplingOutput(extfvk.fvk.ovk, pk, 35000, {}); + auto tx = builder.Build().GetTxOrThrow(); + CWalletTx wtx {&wallet, tx}; - // Simulate SyncTransaction which calls AddToWalletIfInvolvingMe - auto saplingNoteData = wallet.FindMySaplingNotes(wtx).first; - ASSERT_TRUE(saplingNoteData.size() > 0); - wtx.SetSaplingNoteData(saplingNoteData); - wtx.SetMerkleBranch(block); - wallet.AddToWallet(wtx, true, NULL); + // Fake-mine the transaction + EXPECT_EQ(-1, chainActive.Height()); + SproutMerkleTree sproutTree; + CBlock block; + block.vtx.push_back(wtx); + block.hashMerkleRoot = block.BuildMerkleTree(); + auto blockHash = block.GetHash(); + CBlockIndex fakeIndex {block}; + mapBlockIndex.insert(std::make_pair(blockHash, &fakeIndex)); + chainActive.SetTip(&fakeIndex); + EXPECT_TRUE(chainActive.Contains(&fakeIndex)); + EXPECT_EQ(0, chainActive.Height()); - // Simulate receiving new block and ChainTip signal - wallet.IncrementNoteWitnesses(&fakeIndex, &block, sproutTree, saplingTree); - wallet.UpdateSaplingNullifierNoteMapForBlock(&block); + // Simulate SyncTransaction which calls AddToWalletIfInvolvingMe + auto saplingNoteData = wallet.FindMySaplingNotes(wtx, builderHeights[ver]).first; + ASSERT_TRUE(saplingNoteData.size() > 0); + wtx.SetSaplingNoteData(saplingNoteData); + wtx.SetMerkleBranch(block); + wallet.AddToWallet(wtx, true, NULL); - // Retrieve the updated wtx from wallet - uint256 hash = wtx.GetHash(); - wtx = wallet.mapWallet[hash]; + // Simulate receiving new block and ChainTip signal + wallet.IncrementNoteWitnesses(&fakeIndex, &block, sproutTree, saplingTree); + wallet.UpdateSaplingNullifierNoteMapForBlock(&block); - // Decrypt output note B - auto maybe_pt = libzcash::SaplingNotePlaintext::decrypt( - consensusParams, - wtx.nExpiryHeight, - wtx.vShieldedOutput[0].encCiphertext, - ivk, - wtx.vShieldedOutput[0].ephemeralKey, - wtx.vShieldedOutput[0].cmu); - ASSERT_EQ(static_cast(maybe_pt), true); - auto maybe_note = maybe_pt.get().note(ivk); - ASSERT_EQ(static_cast(maybe_note), true); - auto note2 = maybe_note.get(); + // Retrieve the updated wtx from wallet + uint256 hash = wtx.GetHash(); + wtx = wallet.mapWallet[hash]; - SaplingOutPoint sop0(wtx.GetHash(), 0); - auto spend_note_witness = wtx.mapSaplingNoteData[sop0].witnesses.front(); - auto maybe_nf = note2.nullifier(extfvk.fvk, spend_note_witness.position()); - ASSERT_EQ(static_cast(maybe_nf), true); - auto nullifier2 = maybe_nf.get(); + // Decrypt output note B + auto maybe_pt = libzcash::SaplingNotePlaintext::decrypt( + consensusParams, + wtx.nExpiryHeight, + wtx.vShieldedOutput[0].encCiphertext, + ivk, + wtx.vShieldedOutput[0].ephemeralKey, + wtx.vShieldedOutput[0].cmu); + ASSERT_EQ(static_cast(maybe_pt), true); + auto maybe_note = maybe_pt.get().note(ivk); + ASSERT_EQ(static_cast(maybe_note), true); + auto note2 = maybe_note.get(); - anchor = saplingTree.root(); + SaplingOutPoint sop0(wtx.GetHash(), 0); + auto spend_note_witness = wtx.mapSaplingNoteData[sop0].witnesses.front(); + auto maybe_nf = note2.nullifier(extfvk.fvk, spend_note_witness.position()); + ASSERT_EQ(static_cast(maybe_nf), true); + auto nullifier2 = maybe_nf.get(); - // Create transaction to spend note B - auto builder2 = TransactionBuilder(consensusParams, 2); - builder2.AddSaplingSpend(expsk, note2, anchor, spend_note_witness); - builder2.AddSaplingOutput(extfvk.fvk.ovk, pk, 20000, {}); - auto tx2 = builder2.Build().GetTxOrThrow(); + anchor = saplingTree.root(); - // Create conflicting transaction which also spends note B - auto builder3 = TransactionBuilder(consensusParams, 2); - builder3.AddSaplingSpend(expsk, note2, anchor, spend_note_witness); - builder3.AddSaplingOutput(extfvk.fvk.ovk, pk, 19999, {}); - auto tx3 = builder3.Build().GetTxOrThrow(); + // Create transaction to spend note B + auto builder2 = TransactionBuilder(consensusParams, builderHeights[ver] + 1); + builder2.AddSaplingSpend(expsk, note2, anchor, spend_note_witness); + builder2.AddSaplingOutput(extfvk.fvk.ovk, pk, 20000, {}); + auto tx2 = builder2.Build().GetTxOrThrow(); - CWalletTx wtx2 {&wallet, tx2}; - CWalletTx wtx3 {&wallet, tx3}; + // Create conflicting transaction which also spends note B + auto builder3 = TransactionBuilder(consensusParams, builderHeights[ver] + 1); + builder3.AddSaplingSpend(expsk, note2, anchor, spend_note_witness); + builder3.AddSaplingOutput(extfvk.fvk.ovk, pk, 19999, {}); + auto tx3 = builder3.Build().GetTxOrThrow(); - auto hash2 = wtx2.GetHash(); - auto hash3 = wtx3.GetHash(); + CWalletTx wtx2 {&wallet, tx2}; + CWalletTx wtx3 {&wallet, tx3}; - // No conflicts for no spends (wtx is currently the only transaction in the wallet) - EXPECT_EQ(0, wallet.GetConflicts(hash2).size()); - EXPECT_EQ(0, wallet.GetConflicts(hash3).size()); + auto hash2 = wtx2.GetHash(); + auto hash3 = wtx3.GetHash(); - // No conflicts for one spend - wallet.AddToWallet(wtx2, true, NULL); - EXPECT_EQ(0, wallet.GetConflicts(hash2).size()); + // No conflicts for no spends (wtx is currently the only transaction in the wallet) + EXPECT_EQ(0, wallet.GetConflicts(hash2).size()); + EXPECT_EQ(0, wallet.GetConflicts(hash3).size()); - // Conflicts for two spends - wallet.AddToWallet(wtx3, true, NULL); - auto c3 = wallet.GetConflicts(hash2); - EXPECT_EQ(2, c3.size()); - EXPECT_EQ(std::set({hash2, hash3}), c3); + // No conflicts for one spend + wallet.AddToWallet(wtx2, true, NULL); + EXPECT_EQ(0, wallet.GetConflicts(hash2).size()); - // Tear down - chainActive.SetTip(NULL); - mapBlockIndex.erase(blockHash); + // Conflicts for two spends + wallet.AddToWallet(wtx3, true, NULL); + auto c3 = wallet.GetConflicts(hash2); + EXPECT_EQ(2, c3.size()); + EXPECT_EQ(std::set({hash2, hash3}), c3); - // Revert to default - RegtestDeactivateSapling(); + // Tear down + chainActive.SetTip(NULL); + mapBlockIndex.erase(blockHash); + } + + // Revert to test default + RegtestDeactivateCanopy(); + RegtestActivateSapling(); } TEST(WalletTests, SproutNullifierIsSpent) { @@ -930,7 +956,7 @@ TEST(WalletTests, NavigateFromSaplingNullifierToNote) { // Simulate SyncTransaction which calls AddToWalletIfInvolvingMe wtx.SetMerkleBranch(block); - auto saplingNoteData = wallet.FindMySaplingNotes(wtx).first; + auto saplingNoteData = wallet.FindMySaplingNotes(wtx, chainActive.Height()).first; ASSERT_TRUE(saplingNoteData.size() > 0); wtx.SetSaplingNoteData(saplingNoteData); wallet.AddToWallet(wtx, true, NULL); @@ -1007,146 +1033,159 @@ TEST(WalletTests, SpentSproutNoteIsFromMe) { // Create note A, spend A to create note B, spend and verify note B is from me. TEST(WalletTests, SpentSaplingNoteIsFromMe) { - auto consensusParams = RegtestActivateSapling(); + SelectParams(CBaseChainParams::REGTEST); + int overwinterActivationHeight = 5; + int saplingActivationHeight = 30; + int canopyActivationHeight = 70; + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_OVERWINTER, overwinterActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_SAPLING, saplingActivationHeight); + UpdateNetworkUpgradeParameters(Consensus::UPGRADE_CANOPY, canopyActivationHeight); + auto consensusParams = Params().GetConsensus(); - TestWallet wallet; - LOCK2(cs_main, wallet.cs_wallet); + unsigned char leadBytes[] = {0x01, 0x02}; + int builderHeights[] = {saplingActivationHeight, canopyActivationHeight}; - // Generate Sapling address - auto sk = GetTestMasterSaplingSpendingKey(); - auto expsk = sk.expsk; - auto extfvk = sk.ToXFVK(); - auto ivk = extfvk.fvk.in_viewing_key(); - auto pk = sk.DefaultAddress(); + for (int ver = 0; ver < sizeof(leadBytes); ver++) { + TestWallet wallet; + LOCK2(cs_main, wallet.cs_wallet); - // Generate Sapling note A - libzcash::SaplingNote note(pk, 50000, 0x01); - auto cm = note.cmu().get(); - SaplingMerkleTree saplingTree; - saplingTree.append(cm); - auto anchor = saplingTree.root(); - auto witness = saplingTree.witness(); + // Generate Sapling address + auto sk = GetTestMasterSaplingSpendingKey(); + auto expsk = sk.expsk; + auto extfvk = sk.ToXFVK(); + auto ivk = extfvk.fvk.in_viewing_key(); + auto pk = sk.DefaultAddress(); - // Generate transaction, which sends funds to note B - auto builder = TransactionBuilder(consensusParams, 1); - builder.AddSaplingSpend(expsk, note, anchor, witness); - builder.AddSaplingOutput(extfvk.fvk.ovk, pk, 25000, {}); - auto tx = builder.Build().GetTxOrThrow(); + // Generate Sapling note A + libzcash::SaplingNote note(pk, 50000, leadBytes[ver]); + auto cm = note.cmu().get(); + SaplingMerkleTree saplingTree; + saplingTree.append(cm); + auto anchor = saplingTree.root(); + auto witness = saplingTree.witness(); - CWalletTx wtx {&wallet, tx}; - ASSERT_TRUE(wallet.AddSaplingZKey(sk)); - ASSERT_TRUE(wallet.HaveSaplingSpendingKey(extfvk)); + // Generate transaction, which sends funds to note B + auto builder = TransactionBuilder(consensusParams, builderHeights[ver]); + builder.AddSaplingSpend(expsk, note, anchor, witness); + builder.AddSaplingOutput(extfvk.fvk.ovk, pk, 25000, {}); + auto tx = builder.Build().GetTxOrThrow(); - // Fake-mine the transaction - EXPECT_EQ(-1, chainActive.Height()); - SproutMerkleTree sproutTree; - CBlock block; - block.vtx.push_back(wtx); - block.hashMerkleRoot = block.BuildMerkleTree(); - auto blockHash = block.GetHash(); - CBlockIndex fakeIndex {block}; - mapBlockIndex.insert(std::make_pair(blockHash, &fakeIndex)); - chainActive.SetTip(&fakeIndex); - EXPECT_TRUE(chainActive.Contains(&fakeIndex)); - EXPECT_EQ(0, chainActive.Height()); + CWalletTx wtx {&wallet, tx}; + ASSERT_TRUE(wallet.AddSaplingZKey(sk)); + ASSERT_TRUE(wallet.HaveSaplingSpendingKey(extfvk)); - auto saplingNoteData = wallet.FindMySaplingNotes(wtx).first; - ASSERT_TRUE(saplingNoteData.size() > 0); - wtx.SetSaplingNoteData(saplingNoteData); - wtx.SetMerkleBranch(block); - wallet.AddToWallet(wtx, true, NULL); + // Fake-mine the transaction + EXPECT_EQ(-1, chainActive.Height()); + SproutMerkleTree sproutTree; + CBlock block; + block.vtx.push_back(wtx); + block.hashMerkleRoot = block.BuildMerkleTree(); + auto blockHash = block.GetHash(); + CBlockIndex fakeIndex {block}; + mapBlockIndex.insert(std::make_pair(blockHash, &fakeIndex)); + chainActive.SetTip(&fakeIndex); + EXPECT_TRUE(chainActive.Contains(&fakeIndex)); + EXPECT_EQ(0, chainActive.Height()); - // Simulate receiving new block and ChainTip signal. - // This triggers calculation of nullifiers for notes belonging to this wallet - // in the output descriptions of wtx. - wallet.IncrementNoteWitnesses(&fakeIndex, &block, sproutTree, saplingTree); - wallet.UpdateSaplingNullifierNoteMapForBlock(&block); + auto saplingNoteData = wallet.FindMySaplingNotes(wtx, builderHeights[ver]).first; + ASSERT_TRUE(saplingNoteData.size() > 0); + wtx.SetSaplingNoteData(saplingNoteData); + wtx.SetMerkleBranch(block); + wallet.AddToWallet(wtx, true, NULL); - // Retrieve the updated wtx from wallet - wtx = wallet.mapWallet[wtx.GetHash()]; + // Simulate receiving new block and ChainTip signal. + // This triggers calculation of nullifiers for notes belonging to this wallet + // in the output descriptions of wtx. + wallet.IncrementNoteWitnesses(&fakeIndex, &block, sproutTree, saplingTree); + wallet.UpdateSaplingNullifierNoteMapForBlock(&block); - // The test wallet never received the fake note which is being spent, so there - // is no mapping from nullifier to notedata stored in mapSaplingNullifiersToNotes. - // Therefore the wallet does not know the tx belongs to the wallet. - EXPECT_FALSE(wallet.IsFromMe(wtx)); + // Retrieve the updated wtx from wallet + wtx = wallet.mapWallet[wtx.GetHash()]; - // Manually compute the nullifier and check map entry does not exist - auto nf = note.nullifier(extfvk.fvk, witness.position()); - ASSERT_TRUE(nf); - ASSERT_FALSE(wallet.mapSaplingNullifiersToNotes.count(nf.get())); + // The test wallet never received the fake note which is being spent, so there + // is no mapping from nullifier to notedata stored in mapSaplingNullifiersToNotes. + // Therefore the wallet does not know the tx belongs to the wallet. + EXPECT_FALSE(wallet.IsFromMe(wtx)); - // Decrypt note B - auto maybe_pt = libzcash::SaplingNotePlaintext::decrypt( - consensusParams, - wtx.nExpiryHeight, - wtx.vShieldedOutput[0].encCiphertext, - ivk, - wtx.vShieldedOutput[0].ephemeralKey, - wtx.vShieldedOutput[0].cmu); - ASSERT_EQ(static_cast(maybe_pt), true); - auto maybe_note = maybe_pt.get().note(ivk); - ASSERT_EQ(static_cast(maybe_note), true); - auto note2 = maybe_note.get(); + // Manually compute the nullifier and check map entry does not exist + auto nf = note.nullifier(extfvk.fvk, witness.position()); + ASSERT_TRUE(nf); + ASSERT_FALSE(wallet.mapSaplingNullifiersToNotes.count(nf.get())); - // Get witness to retrieve position of note B we want to spend - SaplingOutPoint sop0(wtx.GetHash(), 0); - auto spend_note_witness = wtx.mapSaplingNoteData[sop0].witnesses.front(); - auto maybe_nf = note2.nullifier(extfvk.fvk, spend_note_witness.position()); - ASSERT_EQ(static_cast(maybe_nf), true); - auto nullifier2 = maybe_nf.get(); + // Decrypt note B + auto maybe_pt = libzcash::SaplingNotePlaintext::decrypt( + consensusParams, + wtx.nExpiryHeight, + wtx.vShieldedOutput[0].encCiphertext, + ivk, + wtx.vShieldedOutput[0].ephemeralKey, + wtx.vShieldedOutput[0].cmu); + ASSERT_EQ(static_cast(maybe_pt), true); + auto maybe_note = maybe_pt.get().note(ivk); + ASSERT_EQ(static_cast(maybe_note), true); + auto note2 = maybe_note.get(); - // NOTE: Not updating the anchor results in a core dump. Shouldn't builder just return error? - // *** Error in `./zcash-gtest': double free or corruption (out): 0x00007ffd8755d990 *** - anchor = saplingTree.root(); + // Get witness to retrieve position of note B we want to spend + SaplingOutPoint sop0(wtx.GetHash(), 0); + auto spend_note_witness = wtx.mapSaplingNoteData[sop0].witnesses.front(); + auto maybe_nf = note2.nullifier(extfvk.fvk, spend_note_witness.position()); + ASSERT_EQ(static_cast(maybe_nf), true); + auto nullifier2 = maybe_nf.get(); - // Create transaction to spend note B - auto builder2 = TransactionBuilder(consensusParams, 2); - builder2.AddSaplingSpend(expsk, note2, anchor, spend_note_witness); - builder2.AddSaplingOutput(extfvk.fvk.ovk, pk, 12500, {}); - auto tx2 = builder2.Build().GetTxOrThrow(); - EXPECT_EQ(tx2.vin.size(), 0); - EXPECT_EQ(tx2.vout.size(), 0); - EXPECT_EQ(tx2.vJoinSplit.size(), 0); - EXPECT_EQ(tx2.vShieldedSpend.size(), 1); - EXPECT_EQ(tx2.vShieldedOutput.size(), 2); - EXPECT_EQ(tx2.valueBalance, 10000); + // NOTE: Not updating the anchor results in a core dump. Shouldn't builder just return error? + // *** Error in `./zcash-gtest': double free or corruption (out): 0x00007ffd8755d990 *** + anchor = saplingTree.root(); - CWalletTx wtx2 {&wallet, tx2}; + // Create transaction to spend note B + auto builder2 = TransactionBuilder(consensusParams, builderHeights[ver] + 1); + builder2.AddSaplingSpend(expsk, note2, anchor, spend_note_witness); + builder2.AddSaplingOutput(extfvk.fvk.ovk, pk, 12500, {}); + auto tx2 = builder2.Build().GetTxOrThrow(); + EXPECT_EQ(tx2.vin.size(), 0); + EXPECT_EQ(tx2.vout.size(), 0); + EXPECT_EQ(tx2.vJoinSplit.size(), 0); + EXPECT_EQ(tx2.vShieldedSpend.size(), 1); + EXPECT_EQ(tx2.vShieldedOutput.size(), 2); + EXPECT_EQ(tx2.valueBalance, 10000); - // Fake-mine this tx into the next block - EXPECT_EQ(0, chainActive.Height()); - CBlock block2; - block2.vtx.push_back(wtx2); - block2.hashMerkleRoot = block2.BuildMerkleTree(); - block2.hashPrevBlock = blockHash; - auto blockHash2 = block2.GetHash(); - CBlockIndex fakeIndex2 {block2}; - mapBlockIndex.insert(std::make_pair(blockHash2, &fakeIndex2)); - fakeIndex2.nHeight = 1; - chainActive.SetTip(&fakeIndex2); - EXPECT_TRUE(chainActive.Contains(&fakeIndex2)); - EXPECT_EQ(1, chainActive.Height()); + CWalletTx wtx2 {&wallet, tx2}; - auto saplingNoteData2 = wallet.FindMySaplingNotes(wtx2).first; - ASSERT_TRUE(saplingNoteData2.size() > 0); - wtx2.SetSaplingNoteData(saplingNoteData2); - wtx2.SetMerkleBranch(block2); - wallet.AddToWallet(wtx2, true, NULL); + // Fake-mine this tx into the next block + EXPECT_EQ(0, chainActive.Height()); + CBlock block2; + block2.vtx.push_back(wtx2); + block2.hashMerkleRoot = block2.BuildMerkleTree(); + block2.hashPrevBlock = blockHash; + auto blockHash2 = block2.GetHash(); + CBlockIndex fakeIndex2 {block2}; + mapBlockIndex.insert(std::make_pair(blockHash2, &fakeIndex2)); + fakeIndex2.nHeight = 1; + chainActive.SetTip(&fakeIndex2); + EXPECT_TRUE(chainActive.Contains(&fakeIndex2)); + EXPECT_EQ(1, chainActive.Height()); - // Verify note B is spent. AddToWallet invokes AddToSpends which updates mapTxSaplingNullifiers - EXPECT_TRUE(wallet.IsSaplingSpent(nullifier2)); + auto saplingNoteData2 = wallet.FindMySaplingNotes(wtx2, builderHeights[ver]).first; + ASSERT_TRUE(saplingNoteData2.size() > 0); + wtx2.SetSaplingNoteData(saplingNoteData2); + wtx2.SetMerkleBranch(block2); + wallet.AddToWallet(wtx2, true, NULL); - // Verify note B belongs to wallet. - EXPECT_TRUE(wallet.IsFromMe(wtx2)); - ASSERT_TRUE(wallet.mapSaplingNullifiersToNotes.count(nullifier2)); + // Verify note B is spent. AddToWallet invokes AddToSpends which updates mapTxSaplingNullifiers + EXPECT_TRUE(wallet.IsSaplingSpent(nullifier2)); - // Tear down - chainActive.SetTip(NULL); - mapBlockIndex.erase(blockHash); - mapBlockIndex.erase(blockHash2); + // Verify note B belongs to wallet. + EXPECT_TRUE(wallet.IsFromMe(wtx2)); + ASSERT_TRUE(wallet.mapSaplingNullifiersToNotes.count(nullifier2)); - // Revert to default - RegtestDeactivateSapling(); + // Tear down + chainActive.SetTip(NULL); + mapBlockIndex.erase(blockHash); + mapBlockIndex.erase(blockHash2); + } + + // Revert to test default + RegtestDeactivateCanopy(); + RegtestActivateSapling(); } TEST(WalletTests, CachedWitnessesEmptyChain) { @@ -1847,7 +1886,7 @@ TEST(WalletTests, UpdatedSaplingNoteData) { EXPECT_EQ(0, chainActive.Height()); // Simulate SyncTransaction which calls AddToWalletIfInvolvingMe - auto saplingNoteData = wallet.FindMySaplingNotes(wtx).first; + auto saplingNoteData = wallet.FindMySaplingNotes(wtx, chainActive.Height()).first; ASSERT_TRUE(saplingNoteData.size() == 1); // wallet only has key for change output wtx.SetSaplingNoteData(saplingNoteData); wtx.SetMerkleBranch(block); @@ -1865,7 +1904,7 @@ TEST(WalletTests, UpdatedSaplingNoteData) { ASSERT_TRUE(wallet.AddSaplingZKey(sk2)); ASSERT_TRUE(wallet.HaveSaplingSpendingKey(extfvk2)); CWalletTx wtx2 = wtx; - auto saplingNoteData2 = wallet.FindMySaplingNotes(wtx2).first; + auto saplingNoteData2 = wallet.FindMySaplingNotes(wtx2, chainActive.Height()).first; ASSERT_TRUE(saplingNoteData2.size() == 2); wtx2.SetSaplingNoteData(saplingNoteData2); @@ -1994,7 +2033,7 @@ TEST(WalletTests, MarkAffectedSaplingTransactionsDirty) { EXPECT_EQ(0, chainActive.Height()); // Simulate SyncTransaction which calls AddToWalletIfInvolvingMe - auto saplingNoteData = wallet.FindMySaplingNotes(wtx).first; + auto saplingNoteData = wallet.FindMySaplingNotes(wtx, chainActive.Height()).first; ASSERT_TRUE(saplingNoteData.size() > 0); wtx.SetSaplingNoteData(saplingNoteData); wtx.SetMerkleBranch(block); From 73fae6bf0a57dc627cc5196d0c7abb87b76c9d6e Mon Sep 17 00:00:00 2001 From: Larry Ruane Date: Thu, 25 Jun 2020 09:32:36 -0600 Subject: [PATCH 092/725] review, cleanup: eliminate uninitialized variable --- src/wallet/wallet.cpp | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 5f6ea468a..13b0adf56 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -4654,20 +4654,14 @@ bool CWallet::InitLoadWallet(bool clearWitnessCaches) RegisterValidationInterface(walletInstance); - CBlockIndex *pindexRescan; - if (clearWitnessCaches || GetBoolArg("-rescan", false)) - { + CBlockIndex *pindexRescan = chainActive.Genesis(); + if (clearWitnessCaches || GetBoolArg("-rescan", false)) { walletInstance->ClearNoteWitnessCache(); - pindexRescan = chainActive.Genesis(); - } - else - { + } else { CWalletDB walletdb(walletFile); CBlockLocator locator; if (walletdb.ReadBestBlock(locator)) pindexRescan = FindForkInGlobalIndex(chainActive, locator); - else - pindexRescan = chainActive.Genesis(); } if (chainActive.Tip() && chainActive.Tip() != pindexRescan) { From efd04b920bbfe9914f869a301ff897b39f48c189 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 6 Jun 2020 09:46:45 +1200 Subject: [PATCH 093/725] Implement zip-207 and zip-214. Add funding streams to consensus parameters. Add funding stream payments to coinbase txns generated by the miner. * Reduce valueBalance for shielded outputs to funding streams. * Ensure we produce binding signatures in any case where shielded outputs go to either a funding stream or the miner. --- qa/pull-tester/rpc-tests.sh | 1 + qa/rpc-tests/coinbase_funding_streams.py | 106 +++++++++++++++++ qa/rpc-tests/mining_shielded_coinbase.py | 7 +- qa/rpc-tests/test_framework/mininode.py | 3 + src/Makefile.am | 2 + src/chainparams.cpp | 40 ++++++- src/chainparams.h | 5 + src/consensus/funding.cpp | 70 ++++++++++++ src/consensus/funding.h | 37 ++++++ src/consensus/params.cpp | 126 +++++++++++++++++++-- src/consensus/params.h | 99 +++++++++++++++- src/gtest/test_foundersreward.cpp | 40 +++++++ src/init.cpp | 41 +++++++ src/main.cpp | 67 +++++++++-- src/main.h | 7 +- src/miner.cpp | 138 +++++++++++++++++------ src/rpc/mining.cpp | 1 + src/transaction_builder.h | 6 +- 18 files changed, 729 insertions(+), 67 deletions(-) create mode 100755 qa/rpc-tests/coinbase_funding_streams.py create mode 100644 src/consensus/funding.cpp create mode 100644 src/consensus/funding.h diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index b6454a304..1a330c819 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -82,6 +82,7 @@ testScripts=( 'sprout_sapling_migration.py' 'turnstile.py' 'mining_shielded_coinbase.py' + 'coinbase_funding_streams.py' 'framework.py' 'sapling_rewind_check.py' 'feature_zip221.py' diff --git a/qa/rpc-tests/coinbase_funding_streams.py b/qa/rpc-tests/coinbase_funding_streams.py new file mode 100755 index 000000000..b5c3bf7be --- /dev/null +++ b/qa/rpc-tests/coinbase_funding_streams.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# Copyright (c) 2019 The Zcash developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://www.opensource.org/licenses/mit-license.php . + +from decimal import Decimal +from test_framework.authproxy import JSONRPCException +from test_framework.test_framework import BitcoinTestFramework +from test_framework.mininode import ( + nuparams, + fundingstream, +) +from test_framework.util import ( + assert_equal, + assert_raises, + bitcoind_processes, + connect_nodes, + initialize_chain_clean, + start_node, + wait_and_assert_operationid_status, + check_node_log, + BLOSSOM_BRANCH_ID, + HEARTWOOD_BRANCH_ID, + CANOPY_BRANCH_ID, +) + +class CoinbaseFundingStreamsTest (BitcoinTestFramework): + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def start_node_with(self, index, extra_args=[]): + args = [ + nuparams(BLOSSOM_BRANCH_ID, 1), + nuparams(HEARTWOOD_BRANCH_ID, 2), + nuparams(CANOPY_BRANCH_ID, 5), + "-nurejectoldversions=false", + ] + return start_node(index, self.options.tmpdir, args + extra_args) + + def setup_network(self, split=False): + self.nodes = [] + self.nodes.append(self.start_node_with(0)) + self.nodes.append(self.start_node_with(1)) + connect_nodes(self.nodes[1], 0) + self.is_network_split=False + self.sync_all() + + def run_test (self): + # Generate a shielded address for node 1 for miner rewards, + miner_addr = self.nodes[1].z_getnewaddress('sapling') + + # Generate a shielded address (belonging to node 0) for funding stream + # rewards. + fs_addr = self.nodes[0].z_getnewaddress('sapling') + + # Generate past heartwood activation we won't need node 1 from this + # point onward except to check miner reward balances + self.nodes[1].generate(2) + self.sync_all() + + # Restart node 0 with funding streams. + self.nodes[0].stop() + bitcoind_processes[0].wait() + self.nodes[0] = self.start_node_with(0, [ + "-mineraddress=%s" % miner_addr, + "-minetolocalwallet=0", + fundingstream(0, 5, 9, [fs_addr, fs_addr, fs_addr]), + fundingstream(1, 5, 9, [fs_addr, fs_addr, fs_addr]), + fundingstream(2, 5, 9, [fs_addr, fs_addr, fs_addr]), + ]) + connect_nodes(self.nodes[1], 0) + self.sync_all() + + print("Generate to just prior to Canopy activation") + self.nodes[0].generate(2) + self.sync_all() + + # All miner addresses belong to node 1; check balances + walletinfo = self.nodes[1].getwalletinfo() + assert_equal(walletinfo['immature_balance'], 10) + assert_equal(walletinfo['balance'], 0) + assert_equal(self.nodes[1].z_getbalance(miner_addr, 0), 10) + assert_equal(self.nodes[1].z_getbalance(miner_addr), 10) + + print("Activating Canopy") + self.nodes[0].generate(4) + self.sync_all() + + # check that miner payments made it to node 1's wallet + walletinfo = self.nodes[1].getwalletinfo() + assert_equal(walletinfo['immature_balance'], 10) + assert_equal(walletinfo['balance'], 0) + assert_equal(self.nodes[1].z_getbalance(miner_addr, 0), 30) + assert_equal(self.nodes[1].z_getbalance(miner_addr), 30) + + # check that the node 0 private balance has been augmented by the + # funding stream payments + assert_equal(self.nodes[0].z_getbalance(fs_addr, 0), 5) + assert_equal(self.nodes[0].z_getbalance(fs_addr), 5) + assert_equal(self.nodes[0].z_gettotalbalance()['private'], '5.00') + assert_equal(self.nodes[0].z_gettotalbalance()['total'], '5.00') + +if __name__ == '__main__': + CoinbaseFundingStreamsTest().main() + diff --git a/qa/rpc-tests/mining_shielded_coinbase.py b/qa/rpc-tests/mining_shielded_coinbase.py index 64b73c772..bef3493be 100755 --- a/qa/rpc-tests/mining_shielded_coinbase.py +++ b/qa/rpc-tests/mining_shielded_coinbase.py @@ -6,7 +6,10 @@ from decimal import Decimal from test_framework.authproxy import JSONRPCException from test_framework.test_framework import BitcoinTestFramework +from test_framework.mininode import nuparams from test_framework.util import ( + BLOSSOM_BRANCH_ID, + HEARTWOOD_BRANCH_ID, assert_equal, assert_raises, bitcoind_processes, @@ -25,8 +28,8 @@ class ShieldCoinbaseTest (BitcoinTestFramework): def start_node_with(self, index, extra_args=[]): args = [ - "-nuparams=2bb40e60:1", # Blossom - "-nuparams=f5b9230b:10", # Heartwood + nuparams(BLOSSOM_BRANCH_ID, 1), + nuparams(HEARTWOOD_BRANCH_ID, 10), "-nurejectoldversions=false", ] return start_node(index, self.options.tmpdir, args + extra_args) diff --git a/qa/rpc-tests/test_framework/mininode.py b/qa/rpc-tests/test_framework/mininode.py index 32eceeb79..ca55f6d9b 100755 --- a/qa/rpc-tests/test_framework/mininode.py +++ b/qa/rpc-tests/test_framework/mininode.py @@ -80,6 +80,9 @@ def hash256(s): def nuparams(branch_id, height): return '-nuparams=%x:%d' % (branch_id, height) +def fundingstream(idx, start_height, end_height, addrs): + return '-fundingstream=%d:%d:%d:%s' % (idx, start_height, end_height, ",".join(addrs)) + def ser_compactsize(n): if n < 253: return struct.pack("B", n) diff --git a/src/Makefile.am b/src/Makefile.am index 3b911af92..a86943e80 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -181,6 +181,7 @@ BITCOIN_CORE_H = \ compat/sanity.h \ compressor.h \ consensus/consensus.h \ + consensus/funding.h \ consensus/params.h \ consensus/upgrades.h \ consensus/validation.h \ @@ -413,6 +414,7 @@ libbitcoin_common_a_SOURCES = \ chainparams.cpp \ coins.cpp \ compressor.cpp \ + consensus/funding.cpp \ consensus/params.cpp \ consensus/upgrades.cpp \ core_read.cpp \ diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 9ecb1eb52..fdc3cae5c 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -87,7 +87,7 @@ public: consensus.fCoinbaseMustBeShielded = true; consensus.nSubsidySlowStartInterval = 20000; consensus.nPreBlossomSubsidyHalvingInterval = Consensus::PRE_BLOSSOM_HALVING_INTERVAL; - consensus.nPostBlossomSubsidyHalvingInterval = Consensus::POST_BLOSSOM_HALVING_INTERVAL; + consensus.nPostBlossomSubsidyHalvingInterval = POST_BLOSSOM_HALVING_INTERVAL(Consensus::PRE_BLOSSOM_HALVING_INTERVAL); consensus.nMajorityEnforceBlockUpgrade = 750; consensus.nMajorityRejectBlockOutdated = 950; consensus.nMajorityWindow = 4000; @@ -125,6 +125,8 @@ public: consensus.vUpgrades[Consensus::UPGRADE_CANOPY].nActivationHeight = Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT; + consensus.nFundingPeriodLength = consensus.nPostBlossomSubsidyHalvingInterval / 48; + // The best chain should have at least this much work. consensus.nMinimumChainWork = uint256S("000000000000000000000000000000000000000000000000017e73a331fae01c"); @@ -288,7 +290,7 @@ public: consensus.fCoinbaseMustBeShielded = true; consensus.nSubsidySlowStartInterval = 20000; consensus.nPreBlossomSubsidyHalvingInterval = Consensus::PRE_BLOSSOM_HALVING_INTERVAL; - consensus.nPostBlossomSubsidyHalvingInterval = Consensus::POST_BLOSSOM_HALVING_INTERVAL; + consensus.nPostBlossomSubsidyHalvingInterval = POST_BLOSSOM_HALVING_INTERVAL(Consensus::PRE_BLOSSOM_HALVING_INTERVAL); consensus.nMajorityEnforceBlockUpgrade = 51; consensus.nMajorityRejectBlockOutdated = 75; consensus.nMajorityWindow = 400; @@ -330,6 +332,24 @@ public: consensus.vUpgrades[Consensus::UPGRADE_CANOPY].nActivationHeight = Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT; + consensus.nFundingPeriodLength = consensus.nPostBlossomSubsidyHalvingInterval / 48; + + // TODO: This `if` can be removed once canopy activation height is set. + if (consensus.vUpgrades[Consensus::UPGRADE_CANOPY].nActivationHeight != Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT) { + consensus.AddZIP207FundingStream( + Consensus::FS_ZIP214_ECC, + consensus.vUpgrades[Consensus::UPGRADE_CANOPY].nActivationHeight, 2726400, + {/*TODO*/}); + consensus.AddZIP207FundingStream( + Consensus::FS_ZIP214_ZF, + consensus.vUpgrades[Consensus::UPGRADE_CANOPY].nActivationHeight, 2726400, + {/*TODO*/}); + consensus.AddZIP207FundingStream( + Consensus::FS_ZIP214_MG, + consensus.vUpgrades[Consensus::UPGRADE_CANOPY].nActivationHeight, 2726400, + {/*TODO*/}); + } + // On testnet we activate this rule 6 blocks after Blossom activation. From block 299188 and // prior to Blossom activation, the testnet minimum-difficulty threshold was 15 minutes (i.e. // a minimum difficulty block can be mined if no block is mined normally within 15 minutes): @@ -452,7 +472,7 @@ public: consensus.fCoinbaseMustBeShielded = false; consensus.nSubsidySlowStartInterval = 0; consensus.nPreBlossomSubsidyHalvingInterval = Consensus::PRE_BLOSSOM_REGTEST_HALVING_INTERVAL; - consensus.nPostBlossomSubsidyHalvingInterval = Consensus::POST_BLOSSOM_REGTEST_HALVING_INTERVAL; + consensus.nPostBlossomSubsidyHalvingInterval = POST_BLOSSOM_HALVING_INTERVAL(Consensus::PRE_BLOSSOM_REGTEST_HALVING_INTERVAL); consensus.nMajorityEnforceBlockUpgrade = 750; consensus.nMajorityRejectBlockOutdated = 950; consensus.nMajorityWindow = 1000; @@ -490,6 +510,9 @@ public: consensus.vUpgrades[Consensus::UPGRADE_CANOPY].nActivationHeight = Consensus::NetworkUpgrade::NO_ACTIVATION_HEIGHT; + consensus.nFundingPeriodLength = consensus.nPostBlossomSubsidyHalvingInterval / 48; + // Defined funding streams can be enabled with node config flags. + // The best chain should have at least this much work. consensus.nMinimumChainWork = uint256S("0x00"); @@ -553,6 +576,12 @@ public: consensus.vUpgrades[idx].nActivationHeight = nActivationHeight; } + void UpdateFundingStreamParameters(Consensus::FundingStreamIndex idx, Consensus::FundingStream fs) + { + assert(idx >= Consensus::FIRST_FUNDING_STREAM && idx < Consensus::MAX_FUNDING_STREAMS); + consensus.vFundingStreams[idx] = fs; + } + void UpdateRegtestPow(int64_t nPowMaxAdjustDown, int64_t nPowMaxAdjustUp, uint256 powLimit) { consensus.nPowMaxAdjustDown = nPowMaxAdjustDown; @@ -644,6 +673,11 @@ void UpdateNetworkUpgradeParameters(Consensus::UpgradeIndex idx, int nActivation regTestParams.UpdateNetworkUpgradeParameters(idx, nActivationHeight); } +void UpdateFundingStreamParameters(Consensus::FundingStreamIndex idx, Consensus::FundingStream fs) +{ + regTestParams.UpdateFundingStreamParameters(idx, fs); +} + void UpdateRegtestPow(int64_t nPowMaxAdjustDown, int64_t nPowMaxAdjustUp, uint256 powLimit) { regTestParams.UpdateRegtestPow(nPowMaxAdjustDown, nPowMaxAdjustUp, powLimit); } diff --git a/src/chainparams.h b/src/chainparams.h index 7bb38a6ae..11dbacd96 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -158,4 +158,9 @@ void UpdateNetworkUpgradeParameters(Consensus::UpgradeIndex idx, int nActivation void UpdateRegtestPow(int64_t nPowMaxAdjustDown, int64_t nPowMaxAdjustUp, uint256 powLimit); +/** + * Allows modifying the regtest funding stream parameters. + */ +void UpdateFundingStreamParameters(Consensus::FundingStreamIndex idx, Consensus::FundingStream fs); + #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/consensus/funding.cpp b/src/consensus/funding.cpp new file mode 100644 index 000000000..f0a9283ee --- /dev/null +++ b/src/consensus/funding.cpp @@ -0,0 +1,70 @@ +// Copyright (c) 2020 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php . + +#include + +namespace Consensus +{ +/** + * General information about each funding stream. + * Ordered by Consensus::FundingStreamIndex. + */ +const struct FSInfo FundingStreamInfo[Consensus::MAX_FUNDING_STREAMS] = { + { + /*.recipient =*/ "Electric Coin Company", + /*.specification =*/ "https://zips.z.cash/zip-0214", + /*.valueNumerator =*/ 7, + /*.valueDenominator =*/ 100, + }, + { + /*.recipient =*/ "Zcash Foundation", + /*.specification =*/ "https://zips.z.cash/zip-0214", + /*.valueNumerator =*/ 5, + /*.valueDenominator =*/ 100, + }, + { + /*.recipient =*/ "Major Grants", + /*.specification =*/ "https://zips.z.cash/zip-0214", + /*.valueNumerator =*/ 8, + /*.valueDenominator =*/ 100, + } +}; + +CAmount FSInfo::Value(CAmount blockSubsidy) const +{ + // Integer division is floor division for nonnegative integers in C++ + return CAmount((blockSubsidy * valueNumerator) / valueDenominator); +} + +std::set GetActiveFundingStreamShares( + int nHeight, + CAmount blockSubsidy, + const Consensus::Params& params) +{ + std::set> requiredShares; + for (int idx = Consensus::FIRST_FUNDING_STREAM; idx < Consensus::MAX_FUNDING_STREAMS; idx++) { + auto fs = params.vFundingStreams[idx]; + // Funding period is [startHeight, endHeight) + if (fs && nHeight >= fs.get().GetStartHeight() && nHeight < fs.get().GetEndHeight()) { + requiredShares.insert(std::make_pair( + fs.get().RecipientAddress(params, nHeight), + FundingStreamInfo[idx].Value(blockSubsidy))); + } + } + return requiredShares; +}; + +int GetMaxFundingStreamHeight(const Consensus::Params& params) { + int result = 0; + for (int idx = Consensus::FIRST_FUNDING_STREAM; idx < Consensus::MAX_FUNDING_STREAMS; idx++) { + auto fs = params.vFundingStreams[idx]; + if (fs && result < fs.get().GetEndHeight()) { + result = fs.get().GetEndHeight(); + } + } + + return result; +} + +} // namespace Consensus diff --git a/src/consensus/funding.h b/src/consensus/funding.h new file mode 100644 index 000000000..f789e2ec4 --- /dev/null +++ b/src/consensus/funding.h @@ -0,0 +1,37 @@ +// Copyright (c) 2020 The Zcash developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php . + +#ifndef ZCASH_CONSENSUS_FUNDING_H +#define ZCASH_CONSENSUS_FUNDING_H + +#include +#include + +#include + +namespace Consensus +{ + +struct FSInfo { + std::string recipient; + std::string specification; + uint64_t valueNumerator; + uint64_t valueDenominator; + + CAmount Value(CAmount blockSubsidy) const; +}; + +extern const struct FSInfo FundingStreamInfo[]; + +typedef std::pair FundingStreamShare; + +std::set GetActiveFundingStreamShares( + int nHeight, + CAmount blockSubsidy, + const Consensus::Params& params); + +int GetMaxFundingStreamHeight(const Consensus::Params& params); +} // namespace Consensus + +#endif // ZCASH_CONSENSUS_FUNDING_H diff --git a/src/consensus/params.cpp b/src/consensus/params.cpp index 076866add..7cce1e251 100644 --- a/src/consensus/params.cpp +++ b/src/consensus/params.cpp @@ -4,6 +4,9 @@ #include "params.h" +#include +#include +#include