2016-03-19 12:58:06 -07:00
#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
2022-05-11 16:32:39 -07:00
# Copyright (c) 2020-2022 The Zcash developers
2015-08-26 03:05:36 -07:00
# Distributed under the MIT software license, see the accompanying
2020-12-12 16:33:22 -08:00
# file COPYING or https://www.opensource.org/licenses/mit-license.php .
2015-10-10 22:41:19 -07:00
"""
2017-01-31 11:32:49 -08:00
rpc - tests . py - run regression test suite
2015-10-10 22:41:19 -07:00
This module calls down into individual test cases via subprocess . It will
2017-01-31 11:32:49 -08:00
forward all unrecognized arguments onto the individual test scripts .
2015-10-10 22:41:19 -07:00
2017-02-17 11:22:56 -08:00
RPC tests are disabled on Windows by default . Use - - force to run them anyway .
2015-10-10 22:41:19 -07:00
For a description of arguments recognized by test scripts , see
` qa / pull - tester / test_framework / test_framework . py : BitcoinTestFramework . main ` .
"""
2015-08-26 03:05:36 -07:00
2017-01-31 10:15:40 -08:00
import argparse
2017-01-30 14:57:27 -08:00
import configparser
2015-08-26 03:05:36 -07:00
import os
2015-11-30 05:53:07 -08:00
import time
2015-10-10 22:41:19 -07:00
import shutil
2015-08-26 03:05:36 -07:00
import sys
import subprocess
2015-10-10 22:41:19 -07:00
import tempfile
2015-08-26 03:05:36 -07:00
import re
2015-10-10 22:41:19 -07:00
2020-12-02 05:52:08 -08:00
SERIAL_SCRIPTS = [
# These tests involve enough shielded spends (consuming all CPU
# cores) that we can't run them in parallel.
' mergetoaddress_sapling.py ' ,
' wallet_shieldingcoinbase.py ' ,
]
2017-01-31 10:15:40 -08:00
BASE_SCRIPTS = [
2017-02-06 06:07:14 -08:00
# Scripts that are run by the travis build process
2017-02-17 11:22:56 -08:00
# Longest test should go first, to favor running tests in parallel
2016-12-03 12:46:33 -08:00
# vv Tests less than 5m vv
' wallet.py ' ,
' sprout_sapling_migration.py ' ,
' remove_sprout_shielding.py ' ,
2015-07-15 11:47:45 -07:00
' mempool_packages.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 2m vv
' mergetoaddress_mixednotes.py ' ,
' wallet_shieldcoinbase_sapling.py ' ,
2022-01-24 22:22:15 -08:00
' wallet_shieldcoinbase_ua_sapling.py ' ,
' wallet_shieldcoinbase_ua_nu5.py ' ,
2016-12-03 12:46:33 -08:00
' turnstile.py ' ,
2016-04-29 03:51:15 -07:00
' walletbackup.py ' ,
2016-12-03 12:46:33 -08:00
' zkey_import_export.py ' ,
2015-08-26 03:05:36 -07:00
' prioritisetransaction.py ' ,
' wallet_changeaddresses.py ' ,
' wallet_listreceived.py ' ,
2016-12-03 12:46:33 -08:00
' mempool_tx_expiry.py ' ,
' finalsaplingroot.py ' ,
2022-09-13 07:43:40 -07:00
' finalorchardroot.py ' ,
2022-03-29 17:58:13 -07:00
' wallet_orchard.py ' ,
2015-08-26 03:05:36 -07:00
' wallet_overwintertx.py ' ,
' wallet_persistence.py ' ,
2016-12-03 12:46:33 -08:00
' wallet_listnotes.py ' ,
2022-09-21 19:10:28 -07:00
' wallet_listunspent.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 60s vv
2022-03-02 17:45:30 -08:00
' orchard_reorg.py ' ,
2016-12-03 12:46:33 -08:00
' fundrawtransaction.py ' ,
' reorg_limit.py ' ,
' mempool_limit.py ' ,
' p2p-fullblocktest.py ' ,
# vv Tests less than 30s vv
2015-08-26 03:05:36 -07:00
' wallet_1941.py ' ,
2022-01-13 10:45:40 -08:00
' wallet_accounts.py ' ,
2015-08-26 03:05:36 -07:00
' wallet_addresses.py ' ,
2016-12-03 12:46:33 -08:00
' wallet_anchorfork.py ' ,
' wallet_changeindicator.py ' ,
2022-04-21 15:47:56 -07:00
' wallet_deprecation.py ' ,
2022-03-28 14:25:36 -07:00
' wallet_doublespend.py ' ,
2016-12-03 12:46:33 -08:00
' wallet_import_export.py ' ,
2021-09-27 10:42:02 -07:00
' wallet_isfromme.py ' ,
2022-03-30 11:15:16 -07:00
' wallet_orchard_change.py ' ,
2022-03-31 14:46:41 -07:00
' wallet_orchard_init.py ' ,
2022-03-29 17:58:13 -07:00
' wallet_orchard_persistence.py ' ,
2016-12-03 12:46:33 -08:00
' wallet_nullifiers.py ' ,
2015-08-26 03:05:36 -07:00
' wallet_sapling.py ' ,
2016-12-03 12:46:33 -08:00
' wallet_sendmany_any_taddr.py ' ,
' wallet_treestate.py ' ,
2022-04-01 11:34:32 -07:00
' wallet_unified_change.py ' ,
2015-08-26 03:05:36 -07:00
' listtransactions.py ' ,
' mempool_resurrect_test.py ' ,
' txn_doublespend.py ' ,
' txn_doublespend.py --mineblock ' ,
' getchaintips.py ' ,
' rawtransactions.py ' ,
' getrawtransaction_insight.py ' ,
' rest.py ' ,
' mempool_spendcoinbase.py ' ,
' mempool_reorg.py ' ,
' mempool_nu_activation.py ' ,
' httpbasics.py ' ,
' multi_rpc.py ' ,
' zapwallettxes.py ' ,
' proxy_test.py ' ,
' merkle_blocks.py ' ,
' signrawtransactions.py ' ,
' signrawtransaction_offline.py ' ,
' key_import_export.py ' ,
' nodehandling.py ' ,
' reindex.py ' ,
' addressindex.py ' ,
' spentindex.py ' ,
' timestampindex.py ' ,
' decodescript.py ' ,
' blockchain.py ' ,
' disablewallet.py ' ,
2015-11-30 05:53:07 -08:00
' keypool.py ' ,
2015-08-26 03:05:36 -07:00
' getblocktemplate.py ' ,
2019-02-11 08:59:34 -08:00
' getmininginfo.py ' ,
2015-08-26 03:05:36 -07:00
' bip65-cltv-p2p.py ' ,
' bipdersig-p2p.py ' ,
2015-12-08 08:10:41 -08:00
' invalidblockrequest.py ' ,
' invalidtxrequest.py ' ,
2015-08-26 03:05:36 -07:00
' p2p_nu_peer_management.py ' ,
' rewind_index.py ' ,
' p2p_txexpiry_dos.py ' ,
' p2p_txexpiringsoon.py ' ,
' p2p_node_bloom.py ' ,
' regtest_signrawtransaction.py ' ,
' shorter_block_times.py ' ,
' mining_shielded_coinbase.py ' ,
' coinbase_funding_streams.py ' ,
' framework.py ' ,
' sapling_rewind_check.py ' ,
' feature_zip221.py ' ,
2021-09-08 07:40:47 -07:00
' feature_zip239.py ' ,
2021-06-15 10:52:39 -07:00
' feature_zip244_blockcommitments.py ' ,
2015-08-26 03:05:36 -07:00
' upgrade_golden.py ' ,
2021-12-03 12:32:40 -08:00
' nuparams.py ' ,
2015-08-26 03:05:36 -07:00
' post_heartwood_rollback.py ' ,
' feature_logging.py ' ,
' feature_walletfile.py ' ,
2020-12-29 09:42:46 -08:00
' wallet_parsing_amounts.py ' ,
' wallet_broadcast.py ' ,
' wallet_z_sendmany.py ' ,
' wallet_zero_value.py ' ,
2022-05-06 10:59:28 -07:00
' threeofthreerestore.py ' ,
2022-10-17 09:06:03 -07:00
' show_help.py ' ,
2015-08-26 03:05:36 -07:00
]
2017-02-06 06:07:14 -08:00
ZMQ_SCRIPTS = [
# ZMQ test can only be run if bitcoin was built with zmq-enabled.
2022-06-22 08:53:17 -07:00
# call rpc_tests.py with --nozmq to explicitly exclude these tests.
2017-02-06 06:07:14 -08:00
" zmq_test.py " ]
2016-04-09 13:17:52 -07:00
2017-01-31 10:15:40 -08:00
EXTENDED_SCRIPTS = [
2017-02-06 06:07:14 -08:00
# These tests are not run by the travis build process.
# Longest test should go first, to favor running tests in parallel
2016-12-03 12:46:33 -08:00
' pruning.py ' ,
# vv Tests less than 20m vv
' smartfees.py ' ,
# vv Tests less than 5m vv
# vv Tests less than 2m vv
2015-08-26 03:05:36 -07:00
' getblocktemplate_longpoll.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 60s vv
' rpcbind_test.py ' ,
# vv Tests less than 30s vv
2015-08-26 03:05:36 -07:00
' getblocktemplate_proposals.py ' ,
' forknotify.py ' ,
' hardforkdetection.py ' ,
' invalidateblock.py ' ,
' receivedby.py ' ,
' maxblocksinflight.py ' ,
# 'forknotify.py',
' p2p-acceptblock.py ' ,
2015-09-18 12:59:55 -07:00
' maxuploadtarget.py ' ,
2015-08-26 03:05:36 -07:00
' wallet_db_flush.py ' ,
]
2020-12-02 05:52:08 -08:00
ALL_SCRIPTS = SERIAL_SCRIPTS + BASE_SCRIPTS + ZMQ_SCRIPTS + EXTENDED_SCRIPTS
2016-04-29 03:51:15 -07:00
2017-02-06 06:07:14 -08:00
def main ( ) :
# Parse arguments and pass through unrecognised args
parser = argparse . ArgumentParser ( add_help = False ,
usage = ' %(prog)s [rpc-test.py options] [script options] [scripts] ' ,
description = __doc__ ,
epilog = '''
Help text and arguments for individual test script : ''' ,
formatter_class = argparse . RawTextHelpFormatter )
parser . add_argument ( ' --coverage ' , action = ' store_true ' , help = ' generate a basic coverage report for the RPC interface ' )
2017-02-15 06:59:19 -08:00
parser . add_argument ( ' --exclude ' , ' -x ' , help = ' specify a comma-seperated-list of scripts to exclude. Do not include the .py extension in the name. ' )
2017-02-06 06:07:14 -08:00
parser . add_argument ( ' --extended ' , action = ' store_true ' , help = ' run the extended test suite in addition to the basic tests ' )
2017-02-17 11:22:56 -08:00
parser . add_argument ( ' --force ' , ' -f ' , action = ' store_true ' , help = ' run tests even on platforms where they are disabled by default (e.g. windows). ' )
2017-02-06 06:07:14 -08:00
parser . add_argument ( ' --help ' , ' -h ' , ' -? ' , action = ' store_true ' , help = ' print help text and exit ' )
parser . add_argument ( ' --jobs ' , ' -j ' , type = int , default = 4 , help = ' how many test scripts to run in parallel. Default=4. ' )
2022-06-28 13:51:21 -07:00
parser . add_argument ( ' --machines ' , ' -m ' , type = int , default = - 1 , help = ' how many machines to shard the tests over. must also provide individual shard index. Default=-1 (no sharding). ' )
parser . add_argument ( ' --rpcgroup ' , ' -r ' , type = int , default = - 1 , help = ' individual shard index. must also provide how many machines to shard the tests over. Default=-1 (no sharding). ' )
2017-02-17 11:22:56 -08:00
parser . add_argument ( ' --nozmq ' , action = ' store_true ' , help = ' do not run the zmq tests ' )
args , unknown_args = parser . parse_known_args ( )
2017-02-06 06:07:14 -08:00
# Create a set to store arguments and create the passon string
tests = set ( arg for arg in unknown_args if arg [ : 2 ] != " -- " )
passon_args = [ arg for arg in unknown_args if arg [ : 2 ] == " -- " ]
# Read config generated by configure.
config = configparser . ConfigParser ( )
config . read_file ( open ( os . path . dirname ( __file__ ) + " /tests_config.ini " ) )
2017-02-17 11:22:56 -08:00
enable_wallet = config [ " components " ] . getboolean ( " ENABLE_WALLET " )
enable_utils = config [ " components " ] . getboolean ( " ENABLE_UTILS " )
enable_bitcoind = config [ " components " ] . getboolean ( " ENABLE_BITCOIND " )
enable_zmq = config [ " components " ] . getboolean ( " ENABLE_ZMQ " ) and not args . nozmq
2017-02-06 06:07:14 -08:00
2017-02-17 11:22:56 -08:00
if config [ " environment " ] [ " EXEEXT " ] == " .exe " and not args . force :
2017-02-06 06:07:14 -08:00
# https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
# https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
2017-02-17 11:22:56 -08:00
print ( " Tests currently disabled on Windows by default. Use --force option to enable " )
2017-02-06 06:07:14 -08:00
sys . exit ( 0 )
if not ( enable_wallet and enable_utils and enable_bitcoind ) :
print ( " No rpc tests to run. Wallet, utils, and bitcoind must all be enabled " )
print ( " Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make " )
sys . exit ( 0 )
# python3-zmq may not be installed. Handle this gracefully and with some helpful info
if enable_zmq :
try :
import zmq
2020-12-01 17:01:50 -08:00
zmq # Silences pyflakes
2017-02-06 06:07:14 -08:00
except ImportError :
2022-06-22 08:53:17 -07:00
print ( " ERROR: \" import zmq \" failed. Use --nozmq to run without the ZMQ tests. "
2017-02-06 06:07:14 -08:00
" To run zmq tests, see dependency info in /qa/README.md. " )
raise
2017-01-31 10:15:40 -08:00
# Build list of tests
2017-02-17 11:22:56 -08:00
if tests :
2017-01-31 10:15:40 -08:00
# Individual tests have been specified. Run specified tests that exist
# in the ALL_SCRIPTS list. Accept the name with or without .py extension.
test_list = [ t for t in ALL_SCRIPTS if
( t in tests or re . sub ( " .py$ " , " " , t ) in tests ) ]
2020-12-17 11:39:58 -08:00
print ( " Running individually selected tests: " )
for t in test_list :
print ( " \t " + t )
2016-04-27 13:29:52 -07:00
else :
2017-01-31 10:15:40 -08:00
# No individual tests have been specified. Run base tests, and
# optionally ZMQ tests and extended tests.
2020-12-02 05:52:08 -08:00
test_list = SERIAL_SCRIPTS + BASE_SCRIPTS
2017-02-06 06:07:14 -08:00
if enable_zmq :
2017-01-31 10:15:40 -08:00
test_list + = ZMQ_SCRIPTS
if args . extended :
test_list + = EXTENDED_SCRIPTS
# TODO: BASE_SCRIPTS and EXTENDED_SCRIPTS are sorted by runtime
# (for parallel running efficiency). This combined list will is no
# longer sorted.
2017-02-15 06:59:19 -08:00
# Remove the test cases that the user has explicitly asked to exclude.
if args . exclude :
for exclude_test in args . exclude . split ( ' , ' ) :
if exclude_test + " .py " in test_list :
test_list . remove ( exclude_test + " .py " )
if not test_list :
print ( " No valid test scripts specified. Check that your test is in one "
" of the test lists in rpc-tests.py, or run rpc-tests.py with no arguments to run all tests " )
sys . exit ( 0 )
2017-01-31 10:15:40 -08:00
if args . help :
2017-01-31 11:32:49 -08:00
# Print help for rpc-tests.py, then print help of the first script and exit.
parser . print_help ( )
2017-02-06 06:07:14 -08:00
subprocess . check_call ( ( config [ " environment " ] [ " SRCDIR " ] + ' /qa/rpc-tests/ ' + test_list [ 0 ] ) . split ( ) + [ ' -h ' ] )
2016-04-27 13:29:52 -07:00
sys . exit ( 0 )
2022-06-28 13:51:21 -07:00
if ( args . rpcgroup == - 1 ) != ( args . machines == - 1 ) :
print ( " ERROR: Please use both -m and -r options when using parallel rpc_groups. " )
sys . exit ( 0 )
if args . machines == 0 :
print ( " ERROR: -m/--machines must be greater than 0 " )
sys . exit ( 0 )
if args . machines > 0 and ( args . rpcgroup > = args . machines ) :
print ( " ERROR: -r/--rpcgroup must be less than -m/--machines " )
sys . exit ( 0 )
if args . rpcgroup != - 1 and args . machines != - 1 and args . machines > args . rpcgroup :
# Ceiling division using floor division, by inverting the world.
# https://stackoverflow.com/a/17511341
k = - ( len ( test_list ) / / - args . machines )
split_list = list ( test_list [ i * k : ( i + 1 ) * k ] for i in range ( args . machines ) )
tests_to_run = split_list [ args . rpcgroup ]
else :
tests_to_run = test_list
qa: Enable RPC test execution to be overridden from Python
`run_tests` now takes a (subclass of) `RPCTestHandler` as its first
argument, and returns `True` if all tests passed instead of calling
`sys.exit`. This enables RPC tests to be run from Python and the
execution of individual tests to be customised:
```python
import importlib
import sys
sys.path.append('qa/pull-tester')
rpc_tests = importlib.import_module('rpc-tests')
src_dir = '.'
build_dir = '.'
exeext = ''
class MyTestHandler(rpc_tests.RPCTestHandler):
def start_test(self, args, stdout, stderr):
print('Starting test!')
return subprocess.Popen(
args,
universal_newlines=True,
stdout=stdout,
stderr=stderr)
test_list = ['test_to_run.py']
all_passed = rpc_tests.run_tests(MyTestHandler, test_list, src_dir, build_dir, exeext)
```
2023-01-20 03:56:43 -08:00
all_passed = run_tests (
RPCTestHandler ,
tests_to_run ,
config [ " environment " ] [ " SRCDIR " ] ,
config [ " environment " ] [ " BUILDDIR " ] ,
config [ " environment " ] [ " EXEEXT " ] ,
args . jobs ,
args . coverage ,
passon_args )
sys . exit ( not all_passed )
2017-02-06 06:07:14 -08:00
qa: Enable RPC test execution to be overridden from Python
`run_tests` now takes a (subclass of) `RPCTestHandler` as its first
argument, and returns `True` if all tests passed instead of calling
`sys.exit`. This enables RPC tests to be run from Python and the
execution of individual tests to be customised:
```python
import importlib
import sys
sys.path.append('qa/pull-tester')
rpc_tests = importlib.import_module('rpc-tests')
src_dir = '.'
build_dir = '.'
exeext = ''
class MyTestHandler(rpc_tests.RPCTestHandler):
def start_test(self, args, stdout, stderr):
print('Starting test!')
return subprocess.Popen(
args,
universal_newlines=True,
stdout=stdout,
stderr=stderr)
test_list = ['test_to_run.py']
all_passed = rpc_tests.run_tests(MyTestHandler, test_list, src_dir, build_dir, exeext)
```
2023-01-20 03:56:43 -08:00
def run_tests ( test_handler , test_list , src_dir , build_dir , exeext , jobs = 1 , enable_coverage = False , args = [ ] ) :
2017-02-06 06:07:14 -08:00
BOLD = ( " " , " " )
if os . name == ' posix ' :
# primitive formatting on supported
# terminal via ANSI escape sequences:
BOLD = ( ' \033 [0m ' , ' \033 [1m ' )
#Set env vars
2021-12-08 15:12:33 -08:00
if " ZCASHD " not in os . environ :
os . environ [ " ZCASHD " ] = build_dir + ' /src/zcashd ' + exeext
2017-02-06 06:07:14 -08:00
tests_dir = src_dir + ' /qa/rpc-tests/ '
2015-10-10 22:41:19 -07:00
2017-02-22 01:50:44 -08:00
flags = [ " --srcdir= {} /src " . format ( build_dir ) ] + args
2017-02-06 06:07:14 -08:00
flags . append ( " --cachedir= %s /qa/cache " % build_dir )
if enable_coverage :
2015-10-10 22:41:19 -07:00
coverage = RPCCoverage ( )
2016-04-29 03:51:15 -07:00
flags . append ( coverage . flag )
2017-02-06 06:07:14 -08:00
print ( " Initializing coverage directory at %s \n " % coverage . dir )
else :
coverage = None
2016-04-29 03:51:15 -07:00
2017-01-31 11:32:49 -08:00
if len ( test_list ) > 1 and jobs > 1 :
2016-04-29 03:51:15 -07:00
# Populate cache
2017-02-06 06:07:14 -08:00
subprocess . check_output ( [ tests_dir + ' create_cache.py ' ] + flags )
2016-04-09 13:17:52 -07:00
#Run Tests
2017-02-06 06:07:14 -08:00
all_passed = True
2016-04-29 03:51:15 -07:00
time_sum = 0
time0 = time . time ( )
2017-02-06 06:07:14 -08:00
qa: Enable RPC test execution to be overridden from Python
`run_tests` now takes a (subclass of) `RPCTestHandler` as its first
argument, and returns `True` if all tests passed instead of calling
`sys.exit`. This enables RPC tests to be run from Python and the
execution of individual tests to be customised:
```python
import importlib
import sys
sys.path.append('qa/pull-tester')
rpc_tests = importlib.import_module('rpc-tests')
src_dir = '.'
build_dir = '.'
exeext = ''
class MyTestHandler(rpc_tests.RPCTestHandler):
def start_test(self, args, stdout, stderr):
print('Starting test!')
return subprocess.Popen(
args,
universal_newlines=True,
stdout=stdout,
stderr=stderr)
test_list = ['test_to_run.py']
all_passed = rpc_tests.run_tests(MyTestHandler, test_list, src_dir, build_dir, exeext)
```
2023-01-20 03:56:43 -08:00
job_queue = test_handler ( jobs , tests_dir , test_list , flags )
2017-02-06 06:07:14 -08:00
max_len_name = len ( max ( test_list , key = len ) )
2016-04-29 03:51:15 -07:00
results = BOLD [ 1 ] + " %s | %s | %s \n \n " % ( " TEST " . ljust ( max_len_name ) , " PASSED " , " DURATION " ) + BOLD [ 0 ]
for _ in range ( len ( test_list ) ) :
( name , stdout , stderr , passed , duration ) = job_queue . get_next ( )
all_passed = all_passed and passed
time_sum + = duration
print ( ' \n ' + BOLD [ 1 ] + name + BOLD [ 0 ] + " : " )
2016-09-22 03:41:01 -07:00
print ( ' ' if passed else stdout + ' \n ' , end = ' ' )
print ( ' ' if stderr == ' ' else ' stderr: \n ' + stderr + ' \n ' , end = ' ' )
2016-04-29 03:51:15 -07:00
print ( " Pass: %s %s %s , Duration: %s s \n " % ( BOLD [ 1 ] , passed , BOLD [ 0 ] , duration ) )
2017-02-06 06:07:14 -08:00
results + = " %s | %s | %s s \n " % ( name . ljust ( max_len_name ) , str ( passed ) . ljust ( 6 ) , duration )
2016-04-29 03:51:15 -07:00
results + = BOLD [ 1 ] + " \n %s | %s | %s s (accumulated) " % ( " ALL " . ljust ( max_len_name ) , str ( all_passed ) . ljust ( 6 ) , time_sum ) + BOLD [ 0 ]
print ( results )
print ( " \n Runtime: %s s " % ( int ( time . time ( ) - time0 ) ) )
2016-04-09 13:17:52 -07:00
if coverage :
coverage . report_rpc_coverage ( )
print ( " Cleaning up coverage data " )
coverage . cleanup ( )
2015-10-10 22:41:19 -07:00
qa: Enable RPC test execution to be overridden from Python
`run_tests` now takes a (subclass of) `RPCTestHandler` as its first
argument, and returns `True` if all tests passed instead of calling
`sys.exit`. This enables RPC tests to be run from Python and the
execution of individual tests to be customised:
```python
import importlib
import sys
sys.path.append('qa/pull-tester')
rpc_tests = importlib.import_module('rpc-tests')
src_dir = '.'
build_dir = '.'
exeext = ''
class MyTestHandler(rpc_tests.RPCTestHandler):
def start_test(self, args, stdout, stderr):
print('Starting test!')
return subprocess.Popen(
args,
universal_newlines=True,
stdout=stdout,
stderr=stderr)
test_list = ['test_to_run.py']
all_passed = rpc_tests.run_tests(MyTestHandler, test_list, src_dir, build_dir, exeext)
```
2023-01-20 03:56:43 -08:00
return all_passed
2016-04-29 03:51:15 -07:00
class RPCTestHandler :
"""
Trigger the testscrips passed in via the list .
"""
2017-02-06 06:07:14 -08:00
def __init__ ( self , num_tests_parallel , tests_dir , test_list = None , flags = None ) :
2016-04-29 03:51:15 -07:00
assert ( num_tests_parallel > = 1 )
self . num_jobs = num_tests_parallel
2017-02-06 06:07:14 -08:00
self . tests_dir = tests_dir
2016-04-29 03:51:15 -07:00
self . test_list = test_list
self . flags = flags
self . num_running = 0
2016-11-07 13:33:22 -08:00
# In case there is a graveyard of zombie bitcoinds, we can apply a
# pseudorandom offset to hopefully jump over them.
# (625 is PORT_RANGE/MAX_NODES)
self . portseed_offset = int ( time . time ( ) * 1000 ) % 625
2016-04-29 03:51:15 -07:00
self . jobs = [ ]
qa: Enable RPC test execution to be overridden from Python
`run_tests` now takes a (subclass of) `RPCTestHandler` as its first
argument, and returns `True` if all tests passed instead of calling
`sys.exit`. This enables RPC tests to be run from Python and the
execution of individual tests to be customised:
```python
import importlib
import sys
sys.path.append('qa/pull-tester')
rpc_tests = importlib.import_module('rpc-tests')
src_dir = '.'
build_dir = '.'
exeext = ''
class MyTestHandler(rpc_tests.RPCTestHandler):
def start_test(self, args, stdout, stderr):
print('Starting test!')
return subprocess.Popen(
args,
universal_newlines=True,
stdout=stdout,
stderr=stderr)
test_list = ['test_to_run.py']
all_passed = rpc_tests.run_tests(MyTestHandler, test_list, src_dir, build_dir, exeext)
```
2023-01-20 03:56:43 -08:00
def start_test ( self , args , stdout , stderr ) :
return subprocess . Popen (
args ,
universal_newlines = True ,
stdout = stdout ,
stderr = stderr )
2016-04-29 03:51:15 -07:00
def get_next ( self ) :
while self . num_running < self . num_jobs and self . test_list :
# Add tests
self . num_running + = 1
t = self . test_list . pop ( 0 )
2016-11-07 13:33:22 -08:00
port_seed = [ " --portseed= {} " . format ( len ( self . test_list ) + self . portseed_offset ) ]
2016-09-17 02:47:51 -07:00
log_stdout = tempfile . SpooledTemporaryFile ( max_size = 2 * * 16 )
log_stderr = tempfile . SpooledTemporaryFile ( max_size = 2 * * 16 )
2016-04-29 03:51:15 -07:00
self . jobs . append ( ( t ,
time . time ( ) ,
qa: Enable RPC test execution to be overridden from Python
`run_tests` now takes a (subclass of) `RPCTestHandler` as its first
argument, and returns `True` if all tests passed instead of calling
`sys.exit`. This enables RPC tests to be run from Python and the
execution of individual tests to be customised:
```python
import importlib
import sys
sys.path.append('qa/pull-tester')
rpc_tests = importlib.import_module('rpc-tests')
src_dir = '.'
build_dir = '.'
exeext = ''
class MyTestHandler(rpc_tests.RPCTestHandler):
def start_test(self, args, stdout, stderr):
print('Starting test!')
return subprocess.Popen(
args,
universal_newlines=True,
stdout=stdout,
stderr=stderr)
test_list = ['test_to_run.py']
all_passed = rpc_tests.run_tests(MyTestHandler, test_list, src_dir, build_dir, exeext)
```
2023-01-20 03:56:43 -08:00
self . start_test ( ( self . tests_dir + t ) . split ( ) + self . flags + port_seed ,
log_stdout ,
log_stderr ) ,
2016-09-17 02:47:51 -07:00
log_stdout ,
log_stderr ) )
2020-12-02 05:52:08 -08:00
# Run serial scripts on their own. We always run these first,
# so we won't have added any other jobs yet.
if t in SERIAL_SCRIPTS :
break
2016-04-29 03:51:15 -07:00
if not self . jobs :
2016-05-09 12:29:18 -07:00
raise IndexError ( ' pop from empty list ' )
2016-04-29 03:51:15 -07:00
while True :
# Return first proc that finishes
time . sleep ( .5 )
for j in self . jobs :
2016-09-17 02:47:51 -07:00
( name , time0 , proc , log_out , log_err ) = j
2016-04-29 03:51:15 -07:00
if proc . poll ( ) is not None :
2016-09-17 02:47:51 -07:00
log_out . seek ( 0 ) , log_err . seek ( 0 )
[ stdout , stderr ] = [ l . read ( ) . decode ( ' utf-8 ' ) for l in ( log_out , log_err ) ]
log_out . close ( ) , log_err . close ( )
2016-04-29 03:51:15 -07:00
passed = stderr == " " and proc . returncode == 0
self . num_running - = 1
self . jobs . remove ( j )
return name , stdout , stderr , passed , int ( time . time ( ) - time0 )
print ( ' . ' , end = ' ' , flush = True )
2015-10-10 22:41:19 -07:00
class RPCCoverage ( object ) :
"""
Coverage reporting utilities for pull - tester .
Coverage calculation works by having each test script subprocess write
coverage files into a particular directory . These files contain the RPC
commands invoked during testing , as well as a complete listing of RPC
commands per ` bitcoin - cli help ` ( ` rpc_interface . txt ` ) .
After all tests complete , the commands run are combined and diff ' d against
the complete list to calculate uncovered RPC commands .
See also : qa / rpc - tests / test_framework / coverage . py
"""
def __init__ ( self ) :
self . dir = tempfile . mkdtemp ( prefix = " coverage " )
2016-04-29 03:51:15 -07:00
self . flag = ' --coveragedir= %s ' % self . dir
2015-10-10 22:41:19 -07:00
def report_rpc_coverage ( self ) :
"""
Print out RPC commands that were unexercised by tests .
"""
uncovered = self . _get_uncovered_rpc_commands ( )
if uncovered :
print ( " Uncovered RPC commands: " )
print ( " " . join ( ( " - %s \n " % i ) for i in sorted ( uncovered ) ) )
else :
print ( " All RPC commands covered. " )
def cleanup ( self ) :
return shutil . rmtree ( self . dir )
def _get_uncovered_rpc_commands ( self ) :
"""
Return a set of currently untested RPC commands .
"""
# This is shared from `qa/rpc-tests/test-framework/coverage.py`
2017-02-06 06:07:14 -08:00
reference_filename = ' rpc_interface.txt '
coverage_file_prefix = ' coverage. '
2015-10-10 22:41:19 -07:00
2017-02-06 06:07:14 -08:00
coverage_ref_filename = os . path . join ( self . dir , reference_filename )
2015-10-10 22:41:19 -07:00
coverage_filenames = set ( )
all_cmds = set ( )
covered_cmds = set ( )
if not os . path . isfile ( coverage_ref_filename ) :
raise RuntimeError ( " No coverage reference found " )
2021-04-13 18:19:33 -07:00
with open ( coverage_ref_filename , ' r ' , encoding = ' utf8 ' ) as f :
2015-10-10 22:41:19 -07:00
all_cmds . update ( [ i . strip ( ) for i in f . readlines ( ) ] )
for root , dirs , files in os . walk ( self . dir ) :
for filename in files :
2017-02-06 06:07:14 -08:00
if filename . startswith ( coverage_file_prefix ) :
2015-10-10 22:41:19 -07:00
coverage_filenames . add ( os . path . join ( root , filename ) )
for filename in coverage_filenames :
2021-04-13 18:19:33 -07:00
with open ( filename , ' r ' , encoding = ' utf8 ' ) as f :
2015-10-10 22:41:19 -07:00
covered_cmds . update ( [ i . strip ( ) for i in f . readlines ( ) ] )
return all_cmds - covered_cmds
if __name__ == ' __main__ ' :
2017-02-06 06:07:14 -08:00
main ( )