2016-03-19 12:58:06 -07:00
#!/usr/bin/env python3
2018-01-02 09:12:05 -08:00
# Copyright (c) 2014-2017 The Bitcoin Core developers
2015-08-26 03:05:36 -07:00
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
2017-03-09 06:53:26 -08:00
""" Run regression test suite.
2015-10-10 22:41:19 -07:00
This module calls down into individual test cases via subprocess . It will
2017-01-31 11:32:49 -08:00
forward all unrecognized arguments onto the individual test scripts .
2015-10-10 22:41:19 -07:00
2017-03-09 06:44:57 -08:00
Functional tests are disabled on Windows by default . Use - - force to run them anyway .
2015-10-10 22:41:19 -07:00
For a description of arguments recognized by test scripts , see
2017-03-09 06:53:26 -08:00
` test / functional / test_framework / test_framework . py : BitcoinTestFramework . main ` .
2015-10-10 22:41:19 -07:00
"""
2015-08-26 03:05:36 -07:00
2017-01-31 10:15:40 -08:00
import argparse
2017-11-29 10:25:23 -08:00
from collections import deque
2017-01-30 14:57:27 -08:00
import configparser
2017-05-18 14:33:33 -07:00
import datetime
2015-08-26 03:05:36 -07:00
import os
2015-11-30 05:53:07 -08:00
import time
2015-10-10 22:41:19 -07:00
import shutil
2017-06-08 06:33:52 -07:00
import signal
2015-08-26 03:05:36 -07:00
import sys
import subprocess
2015-10-10 22:41:19 -07:00
import tempfile
2015-08-26 03:05:36 -07:00
import re
2017-02-16 11:00:35 -08:00
import logging
2015-10-10 22:41:19 -07:00
2017-07-25 12:39:39 -07:00
# Formatting. Default colors to empty strings.
BOLD , BLUE , RED , GREY = ( " " , " " ) , ( " " , " " ) , ( " " , " " ) , ( " " , " " )
2017-04-17 10:46:20 -07:00
try :
# Make sure python thinks it can write unicode to its stdout
" \u2713 " . encode ( " utf_8 " ) . decode ( sys . stdout . encoding )
TICK = " ✓ "
CROSS = " ✖ "
CIRCLE = " ○ "
except UnicodeDecodeError :
TICK = " P "
CROSS = " x "
CIRCLE = " o "
2017-04-05 12:19:26 -07:00
if os . name == ' posix ' :
# primitive formatting on supported
# terminal via ANSI escape sequences:
BOLD = ( ' \033 [0m ' , ' \033 [1m ' )
2017-04-20 11:56:37 -07:00
BLUE = ( ' \033 [0m ' , ' \033 [0;34m ' )
2017-07-25 12:39:39 -07:00
RED = ( ' \033 [0m ' , ' \033 [0;31m ' )
GREY = ( ' \033 [0m ' , ' \033 [1;30m ' )
2017-04-05 12:19:26 -07:00
2017-03-22 07:26:02 -07:00
TEST_EXIT_PASSED = 0
TEST_EXIT_SKIPPED = 77
2018-02-14 17:24:42 -08:00
# 20 minutes represented in seconds
TRAVIS_TIMEOUT_DURATION = 20 * 60
2017-11-17 09:54:39 -08:00
BASE_SCRIPTS = [
2017-02-17 11:22:56 -08:00
# Scripts that are run by the travis build process.
# Longest test should go first, to favor running tests in parallel
2018-01-24 15:44:29 -08:00
' wallet_hd.py ' ,
' wallet_backup.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 5m vv
2018-01-24 15:44:29 -08:00
' feature_block.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_fundrawtransaction.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_compactblocks.py ' ,
2018-01-24 15:44:29 -08:00
' feature_segwit.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 2m vv
2018-01-24 15:44:29 -08:00
' wallet_basic.py ' ,
2017-10-20 10:27:55 -07:00
' wallet_labels.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_segwit.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_dump.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_listtransactions.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 60s vv
2018-01-24 15:44:29 -08:00
' p2p_sendheaders.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_zapwallettxes.py ' ,
' wallet_importmulti.py ' ,
2016-12-03 12:46:33 -08:00
' mempool_limit.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_txoutproof.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_listreceivedby.py ' ,
' wallet_abandonconflict.py ' ,
2018-01-24 15:44:29 -08:00
' feature_csv_activation.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_rawtransaction.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_address_types.py ' ,
2018-01-24 15:44:29 -08:00
' feature_reindex.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 30s vv
2018-01-24 15:44:29 -08:00
' wallet_keypool_topup.py ' ,
2018-01-24 15:44:30 -08:00
' interface_zmq.py ' ,
' interface_bitcoin_cli.py ' ,
' mempool_resurrect.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_txn_doublespend.py --mineblock ' ,
' wallet_txn_clone.py ' ,
' wallet_txn_clone.py --segwit ' ,
2018-01-24 15:44:29 -08:00
' rpc_getchaintips.py ' ,
2018-01-24 15:44:30 -08:00
' interface_rest.py ' ,
' mempool_spend_coinbase.py ' ,
2015-08-26 18:15:04 -07:00
' mempool_reorg.py ' ,
2017-03-09 14:14:55 -08:00
' mempool_persist.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_multiwallet.py ' ,
' wallet_multiwallet.py --usecli ' ,
2018-01-24 15:44:30 -08:00
' interface_http.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_users.py ' ,
2018-01-24 15:44:29 -08:00
' feature_proxy.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_signrawtransaction.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_disconnect_ban.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_decodescript.py ' ,
' rpc_blockchain.py ' ,
' rpc_deprecated.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_disable.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_net.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_keypool.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_mempool.py ' ,
2018-01-24 15:44:30 -08:00
' mining_prioritisetransaction.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_invalid_block.py ' ,
' p2p_invalid_tx.py ' ,
2018-01-24 15:44:29 -08:00
' feature_versionbits_warning.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_preciousblock.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_importprunedfunds.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_signmessage.py ' ,
2018-01-24 15:44:29 -08:00
' feature_nulldummy.py ' ,
2017-11-17 09:54:39 -08:00
' mempool_accept.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_import_rescan.py ' ,
2018-01-24 15:44:30 -08:00
' mining_basic.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_bumpfee.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_named_arguments.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_listsinceblock.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_leak.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_encryption.py ' ,
2018-01-24 15:44:29 -08:00
' feature_dersig.py ' ,
' feature_cltv.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_uptime.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_resendwallettransactions.py ' ,
2017-12-12 14:33:39 -08:00
' wallet_fallbackfee.py ' ,
2018-01-24 15:44:29 -08:00
' feature_minchainwork.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_fingerprint.py ' ,
2018-01-24 15:44:29 -08:00
' feature_uacomment.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_unrequested_blocks.py ' ,
2017-11-28 06:38:46 -08:00
' feature_logging.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_node_network_limited.py ' ,
2018-03-08 20:43:55 -08:00
' feature_blocksdir.py ' ,
2018-01-24 15:44:29 -08:00
' feature_config_args.py ' ,
2018-03-30 08:36:38 -07:00
' feature_help.py ' ,
2017-12-20 12:05:31 -08:00
# Don't append tests at the end to avoid merge conflicts
# Put them in a random line within the section that fits their approximate run-time
2015-08-26 03:05:36 -07:00
]
2016-04-09 13:17:52 -07:00
2017-01-31 10:15:40 -08:00
EXTENDED_SCRIPTS = [
2017-02-06 06:07:14 -08:00
# These tests are not run by the travis build process.
# Longest test should go first, to favor running tests in parallel
2018-01-24 15:44:29 -08:00
' feature_pruning.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 20m vv
2018-01-24 15:44:29 -08:00
' feature_fee_estimation.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 5m vv
2018-01-24 15:44:29 -08:00
' feature_maxuploadtarget.py ' ,
2016-12-03 12:46:33 -08:00
' mempool_packages.py ' ,
2018-01-24 15:44:29 -08:00
' feature_dbcrash.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 2m vv
2018-01-24 15:44:29 -08:00
' feature_bip68_sequence.py ' ,
2018-01-24 15:44:30 -08:00
' mining_getblocktemplate_longpoll.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_timeouts.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 60s vv
2018-01-24 15:44:29 -08:00
' feature_bip9_softforks.py ' ,
2018-01-24 15:44:29 -08:00
' p2p_feefilter.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_bind.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 30s vv
2018-01-24 15:44:29 -08:00
' feature_assumevalid.py ' ,
2017-06-15 12:29:53 -07:00
' example_test.py ' ,
2018-01-24 15:44:29 -08:00
' wallet_txn_doublespend.py ' ,
' wallet_txn_clone.py --mineblock ' ,
2018-01-24 15:44:29 -08:00
' feature_notifications.py ' ,
2018-01-24 15:44:29 -08:00
' rpc_invalidateblock.py ' ,
2018-01-24 15:44:29 -08:00
' feature_rbf.py ' ,
2015-08-26 03:05:36 -07:00
]
2017-04-17 15:20:35 -07:00
# Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
2017-03-24 15:36:55 -07:00
ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS
2016-04-29 03:51:15 -07:00
2017-03-27 08:33:00 -07:00
NON_SCRIPTS = [
# These are python files that live in the functional tests directory, but are not test scripts.
" combine_logs.py " ,
" create_cache.py " ,
" test_runner.py " ,
]
2017-02-06 06:07:14 -08:00
def main ( ) :
# Parse arguments and pass through unrecognised args
parser = argparse . ArgumentParser ( add_help = False ,
2017-03-09 06:53:26 -08:00
usage = ' %(prog)s [test_runner.py options] [script options] [scripts] ' ,
2017-02-06 06:07:14 -08:00
description = __doc__ ,
epilog = '''
Help text and arguments for individual test script : ''' ,
formatter_class = argparse . RawTextHelpFormatter )
2017-11-29 10:28:45 -08:00
parser . add_argument ( ' --combinedlogslen ' , ' -c ' , type = int , default = 0 , help = ' print a combined log (of length n lines) from all test nodes and test framework to the console on failure. ' )
2017-02-06 06:07:14 -08:00
parser . add_argument ( ' --coverage ' , action = ' store_true ' , help = ' generate a basic coverage report for the RPC interface ' )
2017-08-15 15:24:39 -07:00
parser . add_argument ( ' --exclude ' , ' -x ' , help = ' specify a comma-separated-list of scripts to exclude. ' )
2017-02-06 06:07:14 -08:00
parser . add_argument ( ' --extended ' , action = ' store_true ' , help = ' run the extended test suite in addition to the basic tests ' )
2017-02-17 11:22:56 -08:00
parser . add_argument ( ' --force ' , ' -f ' , action = ' store_true ' , help = ' run tests even on platforms where they are disabled by default (e.g. windows). ' )
2017-02-06 06:07:14 -08:00
parser . add_argument ( ' --help ' , ' -h ' , ' -? ' , action = ' store_true ' , help = ' print help text and exit ' )
parser . add_argument ( ' --jobs ' , ' -j ' , type = int , default = 4 , help = ' how many test scripts to run in parallel. Default=4. ' )
2017-04-12 08:01:31 -07:00
parser . add_argument ( ' --keepcache ' , ' -k ' , action = ' store_true ' , help = ' the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun. ' )
2017-02-16 11:00:35 -08:00
parser . add_argument ( ' --quiet ' , ' -q ' , action = ' store_true ' , help = ' only print results summary and failure logs ' )
2017-05-18 14:33:33 -07:00
parser . add_argument ( ' --tmpdirprefix ' , ' -t ' , default = tempfile . gettempdir ( ) , help = " Root directory for datadirs " )
2017-02-17 11:22:56 -08:00
args , unknown_args = parser . parse_known_args ( )
2017-02-06 06:07:14 -08:00
2017-05-09 07:39:37 -07:00
# args to be passed on always start with two dashes; tests are the remaining unknown args
tests = [ arg for arg in unknown_args if arg [ : 2 ] != " -- " ]
2017-02-06 06:07:14 -08:00
passon_args = [ arg for arg in unknown_args if arg [ : 2 ] == " -- " ]
# Read config generated by configure.
config = configparser . ConfigParser ( )
2017-03-21 11:47:20 -07:00
configfile = os . path . abspath ( os . path . dirname ( __file__ ) ) + " /../config.ini "
2017-05-01 12:12:49 -07:00
config . read_file ( open ( configfile ) )
passon_args . append ( " --configfile= %s " % configfile )
2017-02-06 06:07:14 -08:00
2017-02-16 11:00:35 -08:00
# Set up logging
logging_level = logging . INFO if args . quiet else logging . DEBUG
logging . basicConfig ( format = ' %(message)s ' , level = logging_level )
2017-05-18 14:33:33 -07:00
# Create base test directory
tmpdir = " %s /bitcoin_test_runner_ %s " % ( args . tmpdirprefix , datetime . datetime . now ( ) . strftime ( " % Y % m %d _ % H % M % S " ) )
os . makedirs ( tmpdir )
logging . debug ( " Temporary test directory at %s " % tmpdir )
2017-02-17 11:22:56 -08:00
enable_wallet = config [ " components " ] . getboolean ( " ENABLE_WALLET " )
enable_utils = config [ " components " ] . getboolean ( " ENABLE_UTILS " )
enable_bitcoind = config [ " components " ] . getboolean ( " ENABLE_BITCOIND " )
2017-02-06 06:07:14 -08:00
2017-02-17 11:22:56 -08:00
if config [ " environment " ] [ " EXEEXT " ] == " .exe " and not args . force :
2017-02-06 06:07:14 -08:00
# https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
# https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
2017-02-17 11:22:56 -08:00
print ( " Tests currently disabled on Windows by default. Use --force option to enable " )
2017-02-06 06:07:14 -08:00
sys . exit ( 0 )
2016-04-29 03:51:15 -07:00
2017-02-06 06:07:14 -08:00
if not ( enable_wallet and enable_utils and enable_bitcoind ) :
2017-03-09 06:44:57 -08:00
print ( " No functional tests to run. Wallet, utils, and bitcoind must all be enabled " )
2017-02-06 06:07:14 -08:00
print ( " Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make " )
sys . exit ( 0 )
2017-01-31 10:15:40 -08:00
# Build list of tests
2018-03-17 11:40:56 -07:00
test_list = [ ]
2017-02-17 11:22:56 -08:00
if tests :
2017-01-31 10:15:40 -08:00
# Individual tests have been specified. Run specified tests that exist
# in the ALL_SCRIPTS list. Accept the name with or without .py extension.
2018-02-14 17:24:42 -08:00
tests = [ re . sub ( " \ .py$ " , " " , test ) + " .py " for test in tests ]
for test in tests :
if test in ALL_SCRIPTS :
test_list . append ( test )
2017-05-09 07:39:37 -07:00
else :
2018-02-14 17:24:42 -08:00
print ( " {} WARNING! {} Test ' {} ' not found in full test list. " . format ( BOLD [ 1 ] , BOLD [ 0 ] , test ) )
elif args . extended :
# Include extended tests
2018-03-17 11:40:56 -07:00
test_list + = ALL_SCRIPTS
2016-04-27 13:29:52 -07:00
else :
2018-02-14 17:24:42 -08:00
# Run base tests only
2018-03-17 11:40:56 -07:00
test_list + = BASE_SCRIPTS
2017-01-31 10:15:40 -08:00
2017-02-15 06:59:19 -08:00
# Remove the test cases that the user has explicitly asked to exclude.
if args . exclude :
2018-02-14 17:24:42 -08:00
exclude_tests = [ re . sub ( " \ .py$ " , " " , test ) + " .py " for test in args . exclude . split ( ' , ' ) ]
for exclude_test in exclude_tests :
2017-05-09 07:39:37 -07:00
if exclude_test in test_list :
test_list . remove ( exclude_test )
else :
print ( " {} WARNING! {} Test ' {} ' not found in current test list. " . format ( BOLD [ 1 ] , BOLD [ 0 ] , exclude_test ) )
2017-02-15 06:59:19 -08:00
if not test_list :
print ( " No valid test scripts specified. Check that your test is in one "
2017-03-09 06:53:26 -08:00
" of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests " )
2017-02-15 06:59:19 -08:00
sys . exit ( 0 )
2017-01-31 10:15:40 -08:00
if args . help :
2017-03-23 10:54:18 -07:00
# Print help for test_runner.py, then print help of the first script (with args removed) and exit.
2017-01-31 11:32:49 -08:00
parser . print_help ( )
2017-12-09 15:39:06 -08:00
subprocess . check_call ( [ sys . executable , os . path . join ( config [ " environment " ] [ " SRCDIR " ] , ' test ' , ' functional ' , test_list [ 0 ] . split ( ) [ 0 ] ) , ' -h ' ] )
2016-04-27 13:29:52 -07:00
sys . exit ( 0 )
2017-03-27 08:33:00 -07:00
check_script_list ( config [ " environment " ] [ " SRCDIR " ] )
2017-11-30 02:11:18 -08:00
check_script_prefixes ( )
2017-03-27 08:33:00 -07:00
2017-04-12 08:01:31 -07:00
if not args . keepcache :
shutil . rmtree ( " %s /test/cache " % config [ " environment " ] [ " BUILDDIR " ] , ignore_errors = True )
2017-11-29 10:28:45 -08:00
run_tests ( test_list , config [ " environment " ] [ " SRCDIR " ] , config [ " environment " ] [ " BUILDDIR " ] , config [ " environment " ] [ " EXEEXT " ] , tmpdir , args . jobs , args . coverage , passon_args , args . combinedlogslen )
2017-02-06 06:07:14 -08:00
2017-11-29 10:28:45 -08:00
def run_tests ( test_list , src_dir , build_dir , exeext , tmpdir , jobs = 1 , enable_coverage = False , args = [ ] , combined_logs_len = 0 ) :
2017-04-12 07:41:13 -07:00
# Warn if bitcoind is already running (unix only)
try :
if subprocess . check_output ( [ " pidof " , " bitcoind " ] ) is not None :
print ( " %s WARNING! %s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention! " % ( BOLD [ 1 ] , BOLD [ 0 ] ) )
except ( OSError , subprocess . SubprocessError ) :
pass
# Warn if there is a cache directory
cache_dir = " %s /test/cache " % build_dir
if os . path . isdir ( cache_dir ) :
print ( " %s WARNING! %s There is a cache directory here: %s . If tests fail unexpectedly, try deleting the cache directory. " % ( BOLD [ 1 ] , BOLD [ 0 ] , cache_dir ) )
2017-02-06 06:07:14 -08:00
#Set env vars
if " BITCOIND " not in os . environ :
os . environ [ " BITCOIND " ] = build_dir + ' /src/bitcoind ' + exeext
2017-07-11 10:01:44 -07:00
os . environ [ " BITCOINCLI " ] = build_dir + ' /src/bitcoin-cli ' + exeext
2017-02-06 06:07:14 -08:00
2017-03-09 06:44:57 -08:00
tests_dir = src_dir + ' /test/functional/ '
2015-10-10 22:41:19 -07:00
2017-02-22 01:50:44 -08:00
flags = [ " --srcdir= {} /src " . format ( build_dir ) ] + args
2017-04-12 07:41:13 -07:00
flags . append ( " --cachedir= %s " % cache_dir )
2017-02-06 06:07:14 -08:00
if enable_coverage :
2015-10-10 22:41:19 -07:00
coverage = RPCCoverage ( )
2016-04-29 03:51:15 -07:00
flags . append ( coverage . flag )
2017-02-16 11:00:35 -08:00
logging . debug ( " Initializing coverage directory at %s " % coverage . dir )
2017-02-06 06:07:14 -08:00
else :
coverage = None
2016-04-29 03:51:15 -07:00
2017-01-31 11:32:49 -08:00
if len ( test_list ) > 1 and jobs > 1 :
2016-04-29 03:51:15 -07:00
# Populate cache
2017-10-13 03:19:20 -07:00
try :
2017-12-09 15:39:06 -08:00
subprocess . check_output ( [ sys . executable , tests_dir + ' create_cache.py ' ] + flags + [ " --tmpdir= %s /cache " % tmpdir ] )
2018-01-19 10:04:03 -08:00
except subprocess . CalledProcessError as e :
sys . stdout . buffer . write ( e . output )
raise
2016-04-09 13:17:52 -07:00
#Run Tests
2017-05-18 14:33:33 -07:00
job_queue = TestHandler ( jobs , tests_dir , tmpdir , test_list , flags )
2018-02-14 17:24:42 -08:00
start_time = time . time ( )
2017-04-05 12:19:26 -07:00
test_results = [ ]
2017-02-06 06:07:14 -08:00
max_len_name = len ( max ( test_list , key = len ) )
2017-04-05 12:19:26 -07:00
2016-04-29 03:51:15 -07:00
for _ in range ( len ( test_list ) ) :
2017-11-29 10:25:23 -08:00
test_result , testdir , stdout , stderr = job_queue . get_next ( )
2017-04-05 12:19:26 -07:00
test_results . append ( test_result )
2017-07-25 12:39:39 -07:00
if test_result . status == " Passed " :
2017-04-05 12:19:26 -07:00
logging . debug ( " \n %s %s %s passed, Duration: %s s " % ( BOLD [ 1 ] , test_result . name , BOLD [ 0 ] , test_result . time ) )
2017-07-25 12:39:39 -07:00
elif test_result . status == " Skipped " :
2017-04-05 12:19:26 -07:00
logging . debug ( " \n %s %s %s skipped " % ( BOLD [ 1 ] , test_result . name , BOLD [ 0 ] ) )
2017-02-16 11:00:35 -08:00
else :
2017-04-05 12:19:26 -07:00
print ( " \n %s %s %s failed, Duration: %s s \n " % ( BOLD [ 1 ] , test_result . name , BOLD [ 0 ] , test_result . time ) )
2017-02-16 11:00:35 -08:00
print ( BOLD [ 1 ] + ' stdout: \n ' + BOLD [ 0 ] + stdout + ' \n ' )
print ( BOLD [ 1 ] + ' stderr: \n ' + BOLD [ 0 ] + stderr + ' \n ' )
2017-11-29 10:28:45 -08:00
if combined_logs_len and os . path . isdir ( testdir ) :
# Print the final `combinedlogslen` lines of the combined logs
print ( ' {} Combine the logs and print the last {} lines ... {} ' . format ( BOLD [ 1 ] , combined_logs_len , BOLD [ 0 ] ) )
2017-11-29 10:25:23 -08:00
print ( ' \n ============ ' )
print ( ' {} Combined log for {} : {} ' . format ( BOLD [ 1 ] , testdir , BOLD [ 0 ] ) )
print ( ' ============ \n ' )
2018-02-14 17:54:41 -08:00
combined_logs , _ = subprocess . Popen ( [ sys . executable , os . path . join ( tests_dir , ' combine_logs.py ' ) , ' -c ' , testdir ] , universal_newlines = True , stdout = subprocess . PIPE ) . communicate ( )
2017-11-29 10:28:45 -08:00
print ( " \n " . join ( deque ( combined_logs . splitlines ( ) , combined_logs_len ) ) )
2017-02-06 06:07:14 -08:00
2018-02-14 17:24:42 -08:00
print_results ( test_results , max_len_name , ( int ( time . time ( ) - start_time ) ) )
2016-04-09 13:17:52 -07:00
if coverage :
coverage . report_rpc_coverage ( )
2017-02-16 11:00:35 -08:00
logging . debug ( " Cleaning up coverage data " )
2016-04-09 13:17:52 -07:00
coverage . cleanup ( )
2015-10-10 22:41:19 -07:00
2017-05-18 14:33:33 -07:00
# Clear up the temp directory if all subdirectories are gone
if not os . listdir ( tmpdir ) :
os . rmdir ( tmpdir )
2017-04-11 10:40:54 -07:00
all_passed = all ( map ( lambda test_result : test_result . was_successful , test_results ) )
2017-04-05 12:19:26 -07:00
2016-04-29 03:51:15 -07:00
sys . exit ( not all_passed )
2017-04-05 12:19:26 -07:00
def print_results ( test_results , max_len_name , runtime ) :
2017-07-25 12:39:39 -07:00
results = " \n " + BOLD [ 1 ] + " %s | %s | %s \n \n " % ( " TEST " . ljust ( max_len_name ) , " STATUS " , " DURATION " ) + BOLD [ 0 ]
2017-04-05 12:19:26 -07:00
2018-03-28 02:55:34 -07:00
test_results . sort ( key = TestResult . sort_key )
2017-04-05 12:19:26 -07:00
all_passed = True
time_sum = 0
for test_result in test_results :
2017-04-11 10:40:54 -07:00
all_passed = all_passed and test_result . was_successful
2017-04-05 12:19:26 -07:00
time_sum + = test_result . time
test_result . padding = max_len_name
results + = str ( test_result )
2017-04-07 06:15:29 -07:00
status = TICK + " Passed " if all_passed else CROSS + " Failed "
2018-03-28 02:17:36 -07:00
if not all_passed :
results + = RED [ 1 ]
2017-07-25 12:39:39 -07:00
results + = BOLD [ 1 ] + " \n %s | %s | %s s (accumulated) \n " % ( " ALL " . ljust ( max_len_name ) , status . ljust ( 9 ) , time_sum ) + BOLD [ 0 ]
2018-03-28 02:17:36 -07:00
if not all_passed :
results + = RED [ 0 ]
2017-04-05 12:19:26 -07:00
results + = " Runtime: %s s \n " % ( runtime )
print ( results )
2017-03-09 06:44:57 -08:00
class TestHandler :
2016-04-29 03:51:15 -07:00
"""
2017-06-30 21:00:51 -07:00
Trigger the test scripts passed in via the list .
2016-04-29 03:51:15 -07:00
"""
2017-05-18 14:33:33 -07:00
def __init__ ( self , num_tests_parallel , tests_dir , tmpdir , test_list = None , flags = None ) :
2016-04-29 03:51:15 -07:00
assert ( num_tests_parallel > = 1 )
self . num_jobs = num_tests_parallel
2017-02-06 06:07:14 -08:00
self . tests_dir = tests_dir
2017-05-18 14:33:33 -07:00
self . tmpdir = tmpdir
2016-04-29 03:51:15 -07:00
self . test_list = test_list
self . flags = flags
self . num_running = 0
2016-11-07 13:33:22 -08:00
# In case there is a graveyard of zombie bitcoinds, we can apply a
# pseudorandom offset to hopefully jump over them.
# (625 is PORT_RANGE/MAX_NODES)
self . portseed_offset = int ( time . time ( ) * 1000 ) % 625
2016-04-29 03:51:15 -07:00
self . jobs = [ ]
def get_next ( self ) :
while self . num_running < self . num_jobs and self . test_list :
# Add tests
self . num_running + = 1
2018-02-14 17:24:42 -08:00
test = self . test_list . pop ( 0 )
2017-05-18 14:33:33 -07:00
portseed = len ( self . test_list ) + self . portseed_offset
portseed_arg = [ " --portseed= {} " . format ( portseed ) ]
2016-09-17 02:47:51 -07:00
log_stdout = tempfile . SpooledTemporaryFile ( max_size = 2 * * 16 )
log_stderr = tempfile . SpooledTemporaryFile ( max_size = 2 * * 16 )
2018-02-14 17:24:42 -08:00
test_argv = test . split ( )
2017-11-29 10:25:23 -08:00
testdir = " {} / {} _ {} " . format ( self . tmpdir , re . sub ( " .py$ " , " " , test_argv [ 0 ] ) , portseed )
tmpdir_arg = [ " --tmpdir= {} " . format ( testdir ) ]
2018-02-14 17:24:42 -08:00
self . jobs . append ( ( test ,
2016-04-29 03:51:15 -07:00
time . time ( ) ,
2017-12-09 15:39:06 -08:00
subprocess . Popen ( [ sys . executable , self . tests_dir + test_argv [ 0 ] ] + test_argv [ 1 : ] + self . flags + portseed_arg + tmpdir_arg ,
2016-04-29 03:51:15 -07:00
universal_newlines = True ,
2016-09-17 02:47:51 -07:00
stdout = log_stdout ,
stderr = log_stderr ) ,
2017-11-29 10:25:23 -08:00
testdir ,
2016-09-17 02:47:51 -07:00
log_stdout ,
log_stderr ) )
2016-04-29 03:51:15 -07:00
if not self . jobs :
2016-05-09 12:29:18 -07:00
raise IndexError ( ' pop from empty list ' )
2016-04-29 03:51:15 -07:00
while True :
# Return first proc that finishes
time . sleep ( .5 )
2018-02-14 17:24:42 -08:00
for job in self . jobs :
( name , start_time , proc , testdir , log_out , log_err ) = job
if os . getenv ( ' TRAVIS ' ) == ' true ' and int ( time . time ( ) - start_time ) > TRAVIS_TIMEOUT_DURATION :
# In travis, timeout individual tests (to stop tests hanging and not providing useful output).
2017-06-08 06:33:52 -07:00
proc . send_signal ( signal . SIGINT )
2016-04-29 03:51:15 -07:00
if proc . poll ( ) is not None :
2016-09-17 02:47:51 -07:00
log_out . seek ( 0 ) , log_err . seek ( 0 )
2018-03-18 15:54:29 -07:00
[ stdout , stderr ] = [ log_file . read ( ) . decode ( ' utf-8 ' ) for log_file in ( log_out , log_err ) ]
2016-09-17 02:47:51 -07:00
log_out . close ( ) , log_err . close ( )
2017-03-22 07:26:02 -07:00
if proc . returncode == TEST_EXIT_PASSED and stderr == " " :
2017-07-25 12:39:39 -07:00
status = " Passed "
2017-03-22 07:26:02 -07:00
elif proc . returncode == TEST_EXIT_SKIPPED :
2017-07-25 12:39:39 -07:00
status = " Skipped "
2017-03-22 07:26:02 -07:00
else :
2017-07-25 12:39:39 -07:00
status = " Failed "
2016-04-29 03:51:15 -07:00
self . num_running - = 1
2018-02-14 17:24:42 -08:00
self . jobs . remove ( job )
2017-04-05 12:19:26 -07:00
2018-02-14 17:24:42 -08:00
return TestResult ( name , status , int ( time . time ( ) - start_time ) ) , testdir , stdout , stderr
2016-04-29 03:51:15 -07:00
print ( ' . ' , end = ' ' , flush = True )
2017-04-05 12:19:26 -07:00
class TestResult ( ) :
def __init__ ( self , name , status , time ) :
self . name = name
self . status = status
self . time = time
self . padding = 0
2018-03-28 02:55:34 -07:00
def sort_key ( self ) :
if self . status == " Passed " :
return 0 , self . name . lower ( )
elif self . status == " Failed " :
return 2 , self . name . lower ( )
elif self . status == " Skipped " :
return 1 , self . name . lower ( )
2017-04-05 12:19:26 -07:00
def __repr__ ( self ) :
2017-07-25 12:39:39 -07:00
if self . status == " Passed " :
2017-04-07 06:15:29 -07:00
color = BLUE
glyph = TICK
2017-07-25 12:39:39 -07:00
elif self . status == " Failed " :
2017-04-20 11:56:37 -07:00
color = RED
glyph = CROSS
2017-07-25 12:39:39 -07:00
elif self . status == " Skipped " :
color = GREY
glyph = CIRCLE
2017-04-06 10:31:47 -07:00
2017-07-25 12:39:39 -07:00
return color [ 1 ] + " %s | %s %s | %s s \n " % ( self . name . ljust ( self . padding ) , glyph , self . status . ljust ( 7 ) , self . time ) + color [ 0 ]
2017-04-05 12:19:26 -07:00
2017-04-11 10:40:54 -07:00
@property
def was_successful ( self ) :
return self . status != " Failed "
2017-04-05 12:19:26 -07:00
2017-11-30 02:11:18 -08:00
def check_script_prefixes ( ) :
2018-01-23 12:16:20 -08:00
""" Check that test scripts start with one of the allowed name prefixes. """
2017-11-30 02:11:18 -08:00
good_prefixes_re = re . compile ( " (example|feature|interface|mempool|mining|p2p|rpc|wallet)_ " )
bad_script_names = [ script for script in ALL_SCRIPTS if good_prefixes_re . match ( script ) is None ]
2018-01-23 12:16:20 -08:00
if bad_script_names :
print ( " %s ERROR: %s %d tests not meeting naming conventions: " % ( BOLD [ 1 ] , BOLD [ 0 ] , len ( bad_script_names ) ) )
2017-11-30 02:11:18 -08:00
print ( " %s " % ( " \n " . join ( sorted ( bad_script_names ) ) ) )
2018-01-23 12:16:20 -08:00
raise AssertionError ( " Some tests are not following naming convention! " )
2017-11-30 02:11:18 -08:00
2017-03-27 08:33:00 -07:00
def check_script_list ( src_dir ) :
""" Check scripts directory.
Check that there are no scripts in the functional tests directory which are
not being run by pull - tester . py . """
script_dir = src_dir + ' /test/functional/ '
2018-03-18 15:54:29 -07:00
python_files = set ( [ test_file for test_file in os . listdir ( script_dir ) if test_file . endswith ( " .py " ) ] )
2017-03-27 08:33:00 -07:00
missed_tests = list ( python_files - set ( map ( lambda x : x . split ( ) [ 0 ] , ALL_SCRIPTS + NON_SCRIPTS ) ) )
if len ( missed_tests ) != 0 :
2017-04-12 07:41:13 -07:00
print ( " %s WARNING! %s The following scripts are not being run: %s . Check the test lists in test_runner.py. " % ( BOLD [ 1 ] , BOLD [ 0 ] , str ( missed_tests ) ) )
if os . getenv ( ' TRAVIS ' ) == ' true ' :
# On travis this warning is an error to prevent merging incomplete commits into master
sys . exit ( 1 )
2015-10-10 22:41:19 -07:00
2017-10-16 18:46:23 -07:00
class RPCCoverage ( ) :
2015-10-10 22:41:19 -07:00
"""
2017-03-09 06:53:26 -08:00
Coverage reporting utilities for test_runner .
2015-10-10 22:41:19 -07:00
Coverage calculation works by having each test script subprocess write
coverage files into a particular directory . These files contain the RPC
commands invoked during testing , as well as a complete listing of RPC
commands per ` bitcoin - cli help ` ( ` rpc_interface . txt ` ) .
After all tests complete , the commands run are combined and diff ' d against
the complete list to calculate uncovered RPC commands .
2017-03-09 06:44:57 -08:00
See also : test / functional / test_framework / coverage . py
2015-10-10 22:41:19 -07:00
"""
def __init__ ( self ) :
self . dir = tempfile . mkdtemp ( prefix = " coverage " )
2016-04-29 03:51:15 -07:00
self . flag = ' --coveragedir= %s ' % self . dir
2015-10-10 22:41:19 -07:00
def report_rpc_coverage ( self ) :
"""
Print out RPC commands that were unexercised by tests .
"""
uncovered = self . _get_uncovered_rpc_commands ( )
if uncovered :
print ( " Uncovered RPC commands: " )
2018-02-14 17:24:42 -08:00
print ( " " . join ( ( " - %s \n " % command ) for command in sorted ( uncovered ) ) )
2015-10-10 22:41:19 -07:00
else :
print ( " All RPC commands covered. " )
def cleanup ( self ) :
return shutil . rmtree ( self . dir )
def _get_uncovered_rpc_commands ( self ) :
"""
Return a set of currently untested RPC commands .
"""
2017-03-09 06:44:57 -08:00
# This is shared from `test/functional/test-framework/coverage.py`
2017-02-06 06:07:14 -08:00
reference_filename = ' rpc_interface.txt '
coverage_file_prefix = ' coverage. '
2015-10-10 22:41:19 -07:00
2017-02-06 06:07:14 -08:00
coverage_ref_filename = os . path . join ( self . dir , reference_filename )
2015-10-10 22:41:19 -07:00
coverage_filenames = set ( )
all_cmds = set ( )
covered_cmds = set ( )
if not os . path . isfile ( coverage_ref_filename ) :
raise RuntimeError ( " No coverage reference found " )
2018-03-18 15:54:29 -07:00
with open ( coverage_ref_filename , ' r ' ) as coverage_ref_file :
all_cmds . update ( [ line . strip ( ) for line in coverage_ref_file . readlines ( ) ] )
2015-10-10 22:41:19 -07:00
for root , dirs , files in os . walk ( self . dir ) :
for filename in files :
2017-02-06 06:07:14 -08:00
if filename . startswith ( coverage_file_prefix ) :
2015-10-10 22:41:19 -07:00
coverage_filenames . add ( os . path . join ( root , filename ) )
for filename in coverage_filenames :
2018-03-18 15:54:29 -07:00
with open ( filename , ' r ' ) as coverage_file :
covered_cmds . update ( [ line . strip ( ) for line in coverage_file . readlines ( ) ] )
2015-10-10 22:41:19 -07:00
return all_cmds - covered_cmds
if __name__ == ' __main__ ' :
2017-02-06 06:07:14 -08:00
main ( )