2016-03-19 12:58:06 -07:00
#!/usr/bin/env python3
# Copyright (c) 2014-2016 The Bitcoin Core developers
2015-08-26 03:05:36 -07:00
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
2017-03-09 06:53:26 -08:00
""" Run regression test suite.
2015-10-10 22:41:19 -07:00
This module calls down into individual test cases via subprocess . It will
2017-01-31 11:32:49 -08:00
forward all unrecognized arguments onto the individual test scripts .
2015-10-10 22:41:19 -07:00
2017-03-09 06:44:57 -08:00
Functional tests are disabled on Windows by default . Use - - force to run them anyway .
2015-10-10 22:41:19 -07:00
For a description of arguments recognized by test scripts , see
2017-03-09 06:53:26 -08:00
` test / functional / test_framework / test_framework . py : BitcoinTestFramework . main ` .
2015-10-10 22:41:19 -07:00
"""
2015-08-26 03:05:36 -07:00
2017-01-31 10:15:40 -08:00
import argparse
2017-01-30 14:57:27 -08:00
import configparser
2015-08-26 03:05:36 -07:00
import os
2015-11-30 05:53:07 -08:00
import time
2015-10-10 22:41:19 -07:00
import shutil
2015-08-26 03:05:36 -07:00
import sys
import subprocess
2015-10-10 22:41:19 -07:00
import tempfile
2015-08-26 03:05:36 -07:00
import re
2017-02-16 11:00:35 -08:00
import logging
2015-10-10 22:41:19 -07:00
2017-04-07 06:15:29 -07:00
# Formatting. Default colors to empty strings.
BOLD , BLUE , RED , GREY = ( " " , " " ) , ( " " , " " ) , ( " " , " " ) , ( " " , " " )
2017-04-17 10:46:20 -07:00
try :
# Make sure python thinks it can write unicode to its stdout
" \u2713 " . encode ( " utf_8 " ) . decode ( sys . stdout . encoding )
TICK = " ✓ "
CROSS = " ✖ "
CIRCLE = " ○ "
except UnicodeDecodeError :
TICK = " P "
CROSS = " x "
CIRCLE = " o "
2017-04-05 12:19:26 -07:00
if os . name == ' posix ' :
# primitive formatting on supported
# terminal via ANSI escape sequences:
BOLD = ( ' \033 [0m ' , ' \033 [1m ' )
2017-04-07 06:15:29 -07:00
BLUE = ( ' \033 [0m ' , ' \033 [0;34m ' )
RED = ( ' \033 [0m ' , ' \033 [0;31m ' )
GREY = ( ' \033 [0m ' , ' \033 [1;30m ' )
2017-04-05 12:19:26 -07:00
2017-03-22 07:26:02 -07:00
TEST_EXIT_PASSED = 0
TEST_EXIT_SKIPPED = 77
2017-01-31 10:15:40 -08:00
BASE_SCRIPTS = [
2017-02-17 11:22:56 -08:00
# Scripts that are run by the travis build process.
# Longest test should go first, to favor running tests in parallel
2016-12-03 12:46:33 -08:00
' wallet-hd.py ' ,
2016-04-29 03:51:15 -07:00
' walletbackup.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 5m vv
' p2p-fullblocktest.py ' ,
' fundrawtransaction.py ' ,
' p2p-compactblocks.py ' ,
' segwit.py ' ,
# vv Tests less than 2m vv
2015-08-26 03:05:36 -07:00
' wallet.py ' ,
2016-08-03 19:28:39 -07:00
' wallet-accounts.py ' ,
2016-12-03 12:46:33 -08:00
' p2p-segwit.py ' ,
2016-07-28 05:59:13 -07:00
' wallet-dump.py ' ,
2015-08-26 03:05:36 -07:00
' listtransactions.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 60s vv
' sendheaders.py ' ,
' zapwallettxes.py ' ,
' importmulti.py ' ,
' mempool_limit.py ' ,
' merkle_blocks.py ' ,
2015-11-14 12:44:35 -08:00
' receivedby.py ' ,
2016-12-03 12:46:33 -08:00
' abandonconflict.py ' ,
' bip68-112-113-p2p.py ' ,
' rawtransactions.py ' ,
' reindex.py ' ,
# vv Tests less than 30s vv
2017-03-24 15:36:55 -07:00
" zmq_test.py " ,
2015-08-26 03:05:36 -07:00
' mempool_resurrect_test.py ' ,
' txn_doublespend.py --mineblock ' ,
' txn_clone.py ' ,
' getchaintips.py ' ,
' rest.py ' ,
' mempool_spendcoinbase.py ' ,
2015-08-26 18:15:04 -07:00
' mempool_reorg.py ' ,
2015-08-26 03:05:36 -07:00
' httpbasics.py ' ,
2015-11-11 07:49:32 -08:00
' multi_rpc.py ' ,
2015-08-26 03:05:36 -07:00
' proxy_test.py ' ,
' signrawtransactions.py ' ,
2017-04-19 08:14:53 -07:00
' disconnect_ban.py ' ,
2015-08-26 03:05:36 -07:00
' decodescript.py ' ,
2015-10-08 01:22:50 -07:00
' blockchain.py ' ,
2015-11-08 23:40:46 -08:00
' disablewallet.py ' ,
2017-03-25 07:29:56 -07:00
' net.py ' ,
2015-11-30 05:53:07 -08:00
' keypool.py ' ,
2016-06-30 05:49:59 -07:00
' p2p-mempool.py ' ,
2015-11-18 17:55:52 -08:00
' prioritise_transaction.py ' ,
2015-12-08 08:10:41 -08:00
' invalidblockrequest.py ' ,
' invalidtxrequest.py ' ,
2016-03-15 09:09:16 -07:00
' p2p-versionbits-warning.py ' ,
2016-08-26 14:05:26 -07:00
' preciousblock.py ' ,
2016-02-18 16:31:12 -08:00
' importprunedfunds.py ' ,
2016-04-29 03:51:15 -07:00
' signmessages.py ' ,
2016-08-31 04:38:23 -07:00
' nulldummy.py ' ,
2016-12-12 07:18:11 -08:00
' import-rescan.py ' ,
2016-12-09 10:45:27 -08:00
' bumpfee.py ' ,
2016-09-25 12:09:13 -07:00
' rpcnamedargs.py ' ,
2017-01-11 03:02:25 -08:00
' listsinceblock.py ' ,
2017-02-07 22:17:58 -08:00
' p2p-leaktests.py ' ,
2015-08-26 03:05:36 -07:00
]
2016-04-09 13:17:52 -07:00
2017-01-31 10:15:40 -08:00
EXTENDED_SCRIPTS = [
2017-02-06 06:07:14 -08:00
# These tests are not run by the travis build process.
# Longest test should go first, to favor running tests in parallel
2016-12-03 12:46:33 -08:00
' pruning.py ' ,
# vv Tests less than 20m vv
' smartfees.py ' ,
# vv Tests less than 5m vv
' maxuploadtarget.py ' ,
' mempool_packages.py ' ,
# vv Tests less than 2m vv
' bip68-sequence.py ' ,
' getblocktemplate_longpoll.py ' ,
2017-02-07 14:43:36 -08:00
' p2p-timeouts.py ' ,
2016-12-03 12:46:33 -08:00
# vv Tests less than 60s vv
2016-03-15 22:30:04 -07:00
' bip9-softforks.py ' ,
2016-12-03 12:46:33 -08:00
' p2p-feefilter.py ' ,
' rpcbind_test.py ' ,
# vv Tests less than 30s vv
2017-02-23 15:00:33 -08:00
' assumevalid.py ' ,
2015-06-28 11:42:17 -07:00
' bip65-cltv.py ' ,
' bip65-cltv-p2p.py ' ,
2015-08-26 03:05:36 -07:00
' bipdersig-p2p.py ' ,
' bipdersig.py ' ,
' getblocktemplate_proposals.py ' ,
' txn_doublespend.py ' ,
' txn_clone.py --mineblock ' ,
' forknotify.py ' ,
' invalidateblock.py ' ,
' p2p-acceptblock.py ' ,
2015-10-30 11:55:32 -07:00
' replace-by-fee.py ' ,
2015-08-26 03:05:36 -07:00
]
2017-04-17 15:20:35 -07:00
# Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
2017-03-24 15:36:55 -07:00
ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS
2016-04-29 03:51:15 -07:00
2017-03-27 08:33:00 -07:00
NON_SCRIPTS = [
# These are python files that live in the functional tests directory, but are not test scripts.
" combine_logs.py " ,
" create_cache.py " ,
" test_runner.py " ,
]
2017-02-06 06:07:14 -08:00
def main ( ) :
# Parse arguments and pass through unrecognised args
parser = argparse . ArgumentParser ( add_help = False ,
2017-03-09 06:53:26 -08:00
usage = ' %(prog)s [test_runner.py options] [script options] [scripts] ' ,
2017-02-06 06:07:14 -08:00
description = __doc__ ,
epilog = '''
Help text and arguments for individual test script : ''' ,
formatter_class = argparse . RawTextHelpFormatter )
parser . add_argument ( ' --coverage ' , action = ' store_true ' , help = ' generate a basic coverage report for the RPC interface ' )
2017-02-15 06:59:19 -08:00
parser . add_argument ( ' --exclude ' , ' -x ' , help = ' specify a comma-seperated-list of scripts to exclude. Do not include the .py extension in the name. ' )
2017-02-06 06:07:14 -08:00
parser . add_argument ( ' --extended ' , action = ' store_true ' , help = ' run the extended test suite in addition to the basic tests ' )
2017-02-17 11:22:56 -08:00
parser . add_argument ( ' --force ' , ' -f ' , action = ' store_true ' , help = ' run tests even on platforms where they are disabled by default (e.g. windows). ' )
2017-02-06 06:07:14 -08:00
parser . add_argument ( ' --help ' , ' -h ' , ' -? ' , action = ' store_true ' , help = ' print help text and exit ' )
parser . add_argument ( ' --jobs ' , ' -j ' , type = int , default = 4 , help = ' how many test scripts to run in parallel. Default=4. ' )
2017-04-12 08:01:31 -07:00
parser . add_argument ( ' --keepcache ' , ' -k ' , action = ' store_true ' , help = ' the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun. ' )
2017-02-16 11:00:35 -08:00
parser . add_argument ( ' --quiet ' , ' -q ' , action = ' store_true ' , help = ' only print results summary and failure logs ' )
2017-02-17 11:22:56 -08:00
args , unknown_args = parser . parse_known_args ( )
2017-02-06 06:07:14 -08:00
# Create a set to store arguments and create the passon string
tests = set ( arg for arg in unknown_args if arg [ : 2 ] != " -- " )
passon_args = [ arg for arg in unknown_args if arg [ : 2 ] == " -- " ]
# Read config generated by configure.
config = configparser . ConfigParser ( )
2017-03-09 06:53:26 -08:00
config . read_file ( open ( os . path . dirname ( __file__ ) + " /config.ini " ) )
2017-02-06 06:07:14 -08:00
2017-02-16 11:00:35 -08:00
# Set up logging
logging_level = logging . INFO if args . quiet else logging . DEBUG
logging . basicConfig ( format = ' %(message)s ' , level = logging_level )
2017-02-17 11:22:56 -08:00
enable_wallet = config [ " components " ] . getboolean ( " ENABLE_WALLET " )
enable_utils = config [ " components " ] . getboolean ( " ENABLE_UTILS " )
enable_bitcoind = config [ " components " ] . getboolean ( " ENABLE_BITCOIND " )
2017-02-06 06:07:14 -08:00
2017-02-17 11:22:56 -08:00
if config [ " environment " ] [ " EXEEXT " ] == " .exe " and not args . force :
2017-02-06 06:07:14 -08:00
# https://github.com/bitcoin/bitcoin/commit/d52802551752140cf41f0d9a225a43e84404d3e9
# https://github.com/bitcoin/bitcoin/pull/5677#issuecomment-136646964
2017-02-17 11:22:56 -08:00
print ( " Tests currently disabled on Windows by default. Use --force option to enable " )
2017-02-06 06:07:14 -08:00
sys . exit ( 0 )
2016-04-29 03:51:15 -07:00
2017-02-06 06:07:14 -08:00
if not ( enable_wallet and enable_utils and enable_bitcoind ) :
2017-03-09 06:44:57 -08:00
print ( " No functional tests to run. Wallet, utils, and bitcoind must all be enabled " )
2017-02-06 06:07:14 -08:00
print ( " Rerun `configure` with -enable-wallet, -with-utils and -with-daemon and rerun make " )
sys . exit ( 0 )
2017-01-31 10:15:40 -08:00
# Build list of tests
2017-02-17 11:22:56 -08:00
if tests :
2017-01-31 10:15:40 -08:00
# Individual tests have been specified. Run specified tests that exist
# in the ALL_SCRIPTS list. Accept the name with or without .py extension.
test_list = [ t for t in ALL_SCRIPTS if
( t in tests or re . sub ( " .py$ " , " " , t ) in tests ) ]
2016-04-27 13:29:52 -07:00
else :
2017-03-24 15:36:55 -07:00
# No individual tests have been specified.
# Run all base tests, and optionally run extended tests.
2017-01-31 10:15:40 -08:00
test_list = BASE_SCRIPTS
if args . extended :
2017-04-17 15:20:35 -07:00
# place the EXTENDED_SCRIPTS first since the three longest ones
# are there and the list is shorter
test_list = EXTENDED_SCRIPTS + test_list
2017-01-31 10:15:40 -08:00
2017-02-15 06:59:19 -08:00
# Remove the test cases that the user has explicitly asked to exclude.
if args . exclude :
for exclude_test in args . exclude . split ( ' , ' ) :
if exclude_test + " .py " in test_list :
test_list . remove ( exclude_test + " .py " )
if not test_list :
print ( " No valid test scripts specified. Check that your test is in one "
2017-03-09 06:53:26 -08:00
" of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests " )
2017-02-15 06:59:19 -08:00
sys . exit ( 0 )
2017-01-31 10:15:40 -08:00
if args . help :
2017-03-23 10:54:18 -07:00
# Print help for test_runner.py, then print help of the first script (with args removed) and exit.
2017-01-31 11:32:49 -08:00
parser . print_help ( )
2017-03-23 10:54:18 -07:00
subprocess . check_call ( [ ( config [ " environment " ] [ " SRCDIR " ] + ' /test/functional/ ' + test_list [ 0 ] . split ( ) [ 0 ] ) ] + [ ' -h ' ] )
2016-04-27 13:29:52 -07:00
sys . exit ( 0 )
2017-03-27 08:33:00 -07:00
check_script_list ( config [ " environment " ] [ " SRCDIR " ] )
2017-04-12 08:01:31 -07:00
if not args . keepcache :
shutil . rmtree ( " %s /test/cache " % config [ " environment " ] [ " BUILDDIR " ] , ignore_errors = True )
2017-02-17 11:22:56 -08:00
run_tests ( test_list , config [ " environment " ] [ " SRCDIR " ] , config [ " environment " ] [ " BUILDDIR " ] , config [ " environment " ] [ " EXEEXT " ] , args . jobs , args . coverage , passon_args )
2017-02-06 06:07:14 -08:00
2017-02-17 11:22:56 -08:00
def run_tests ( test_list , src_dir , build_dir , exeext , jobs = 1 , enable_coverage = False , args = [ ] ) :
2017-04-12 07:41:13 -07:00
# Warn if bitcoind is already running (unix only)
try :
if subprocess . check_output ( [ " pidof " , " bitcoind " ] ) is not None :
print ( " %s WARNING! %s There is already a bitcoind process running on this system. Tests may fail unexpectedly due to resource contention! " % ( BOLD [ 1 ] , BOLD [ 0 ] ) )
except ( OSError , subprocess . SubprocessError ) :
pass
# Warn if there is a cache directory
cache_dir = " %s /test/cache " % build_dir
if os . path . isdir ( cache_dir ) :
print ( " %s WARNING! %s There is a cache directory here: %s . If tests fail unexpectedly, try deleting the cache directory. " % ( BOLD [ 1 ] , BOLD [ 0 ] , cache_dir ) )
2017-02-06 06:07:14 -08:00
#Set env vars
if " BITCOIND " not in os . environ :
os . environ [ " BITCOIND " ] = build_dir + ' /src/bitcoind ' + exeext
2017-03-09 06:44:57 -08:00
tests_dir = src_dir + ' /test/functional/ '
2015-10-10 22:41:19 -07:00
2017-02-22 01:50:44 -08:00
flags = [ " --srcdir= {} /src " . format ( build_dir ) ] + args
2017-04-12 07:41:13 -07:00
flags . append ( " --cachedir= %s " % cache_dir )
2017-02-06 06:07:14 -08:00
if enable_coverage :
2015-10-10 22:41:19 -07:00
coverage = RPCCoverage ( )
2016-04-29 03:51:15 -07:00
flags . append ( coverage . flag )
2017-02-16 11:00:35 -08:00
logging . debug ( " Initializing coverage directory at %s " % coverage . dir )
2017-02-06 06:07:14 -08:00
else :
coverage = None
2016-04-29 03:51:15 -07:00
2017-01-31 11:32:49 -08:00
if len ( test_list ) > 1 and jobs > 1 :
2016-04-29 03:51:15 -07:00
# Populate cache
2017-02-06 06:07:14 -08:00
subprocess . check_output ( [ tests_dir + ' create_cache.py ' ] + flags )
2016-04-09 13:17:52 -07:00
#Run Tests
2017-03-09 06:44:57 -08:00
job_queue = TestHandler ( jobs , tests_dir , test_list , flags )
2017-04-05 12:19:26 -07:00
time0 = time . time ( )
test_results = [ ]
2017-02-06 06:07:14 -08:00
max_len_name = len ( max ( test_list , key = len ) )
2017-04-05 12:19:26 -07:00
2016-04-29 03:51:15 -07:00
for _ in range ( len ( test_list ) ) :
2017-04-05 12:19:26 -07:00
test_result , stdout , stderr = job_queue . get_next ( )
test_results . append ( test_result )
if test_result . status == " Passed " :
logging . debug ( " \n %s %s %s passed, Duration: %s s " % ( BOLD [ 1 ] , test_result . name , BOLD [ 0 ] , test_result . time ) )
elif test_result . status == " Skipped " :
logging . debug ( " \n %s %s %s skipped " % ( BOLD [ 1 ] , test_result . name , BOLD [ 0 ] ) )
2017-02-16 11:00:35 -08:00
else :
2017-04-05 12:19:26 -07:00
print ( " \n %s %s %s failed, Duration: %s s \n " % ( BOLD [ 1 ] , test_result . name , BOLD [ 0 ] , test_result . time ) )
2017-02-16 11:00:35 -08:00
print ( BOLD [ 1 ] + ' stdout: \n ' + BOLD [ 0 ] + stdout + ' \n ' )
print ( BOLD [ 1 ] + ' stderr: \n ' + BOLD [ 0 ] + stderr + ' \n ' )
2017-02-06 06:07:14 -08:00
2017-04-05 12:19:26 -07:00
print_results ( test_results , max_len_name , ( int ( time . time ( ) - time0 ) ) )
2016-04-09 13:17:52 -07:00
if coverage :
coverage . report_rpc_coverage ( )
2017-02-16 11:00:35 -08:00
logging . debug ( " Cleaning up coverage data " )
2016-04-09 13:17:52 -07:00
coverage . cleanup ( )
2015-10-10 22:41:19 -07:00
2017-04-11 10:40:54 -07:00
all_passed = all ( map ( lambda test_result : test_result . was_successful , test_results ) )
2017-04-05 12:19:26 -07:00
2016-04-29 03:51:15 -07:00
sys . exit ( not all_passed )
2017-04-05 12:19:26 -07:00
def print_results ( test_results , max_len_name , runtime ) :
2017-04-06 10:31:47 -07:00
results = " \n " + BOLD [ 1 ] + " %s | %s | %s \n \n " % ( " TEST " . ljust ( max_len_name ) , " STATUS " , " DURATION " ) + BOLD [ 0 ]
2017-04-05 12:19:26 -07:00
test_results . sort ( key = lambda result : result . name . lower ( ) )
all_passed = True
time_sum = 0
for test_result in test_results :
2017-04-11 10:40:54 -07:00
all_passed = all_passed and test_result . was_successful
2017-04-05 12:19:26 -07:00
time_sum + = test_result . time
test_result . padding = max_len_name
results + = str ( test_result )
2017-04-07 06:15:29 -07:00
status = TICK + " Passed " if all_passed else CROSS + " Failed "
results + = BOLD [ 1 ] + " \n %s | %s | %s s (accumulated) \n " % ( " ALL " . ljust ( max_len_name ) , status . ljust ( 9 ) , time_sum ) + BOLD [ 0 ]
2017-04-05 12:19:26 -07:00
results + = " Runtime: %s s \n " % ( runtime )
print ( results )
2017-03-09 06:44:57 -08:00
class TestHandler :
2016-04-29 03:51:15 -07:00
"""
Trigger the testscrips passed in via the list .
"""
2017-02-06 06:07:14 -08:00
def __init__ ( self , num_tests_parallel , tests_dir , test_list = None , flags = None ) :
2016-04-29 03:51:15 -07:00
assert ( num_tests_parallel > = 1 )
self . num_jobs = num_tests_parallel
2017-02-06 06:07:14 -08:00
self . tests_dir = tests_dir
2016-04-29 03:51:15 -07:00
self . test_list = test_list
self . flags = flags
self . num_running = 0
2016-11-07 13:33:22 -08:00
# In case there is a graveyard of zombie bitcoinds, we can apply a
# pseudorandom offset to hopefully jump over them.
# (625 is PORT_RANGE/MAX_NODES)
self . portseed_offset = int ( time . time ( ) * 1000 ) % 625
2016-04-29 03:51:15 -07:00
self . jobs = [ ]
def get_next ( self ) :
while self . num_running < self . num_jobs and self . test_list :
# Add tests
self . num_running + = 1
t = self . test_list . pop ( 0 )
2016-11-07 13:33:22 -08:00
port_seed = [ " --portseed= {} " . format ( len ( self . test_list ) + self . portseed_offset ) ]
2016-09-17 02:47:51 -07:00
log_stdout = tempfile . SpooledTemporaryFile ( max_size = 2 * * 16 )
log_stderr = tempfile . SpooledTemporaryFile ( max_size = 2 * * 16 )
2017-03-23 10:54:18 -07:00
test_argv = t . split ( )
2016-04-29 03:51:15 -07:00
self . jobs . append ( ( t ,
time . time ( ) ,
2017-03-23 10:54:18 -07:00
subprocess . Popen ( [ self . tests_dir + test_argv [ 0 ] ] + test_argv [ 1 : ] + self . flags + port_seed ,
2016-04-29 03:51:15 -07:00
universal_newlines = True ,
2016-09-17 02:47:51 -07:00
stdout = log_stdout ,
stderr = log_stderr ) ,
log_stdout ,
log_stderr ) )
2016-04-29 03:51:15 -07:00
if not self . jobs :
2016-05-09 12:29:18 -07:00
raise IndexError ( ' pop from empty list ' )
2016-04-29 03:51:15 -07:00
while True :
# Return first proc that finishes
time . sleep ( .5 )
for j in self . jobs :
2016-09-17 02:47:51 -07:00
( name , time0 , proc , log_out , log_err ) = j
2016-04-29 03:51:15 -07:00
if proc . poll ( ) is not None :
2016-09-17 02:47:51 -07:00
log_out . seek ( 0 ) , log_err . seek ( 0 )
[ stdout , stderr ] = [ l . read ( ) . decode ( ' utf-8 ' ) for l in ( log_out , log_err ) ]
log_out . close ( ) , log_err . close ( )
2017-03-22 07:26:02 -07:00
if proc . returncode == TEST_EXIT_PASSED and stderr == " " :
status = " Passed "
elif proc . returncode == TEST_EXIT_SKIPPED :
status = " Skipped "
else :
status = " Failed "
2016-04-29 03:51:15 -07:00
self . num_running - = 1
self . jobs . remove ( j )
2017-04-05 12:19:26 -07:00
return TestResult ( name , status , int ( time . time ( ) - time0 ) ) , stdout , stderr
2016-04-29 03:51:15 -07:00
print ( ' . ' , end = ' ' , flush = True )
2017-04-05 12:19:26 -07:00
class TestResult ( ) :
def __init__ ( self , name , status , time ) :
self . name = name
self . status = status
self . time = time
self . padding = 0
def __repr__ ( self ) :
2017-04-06 10:31:47 -07:00
if self . status == " Passed " :
2017-04-07 06:15:29 -07:00
color = BLUE
glyph = TICK
2017-04-06 10:31:47 -07:00
elif self . status == " Failed " :
2017-04-07 06:15:29 -07:00
color = RED
glyph = CROSS
2017-04-06 10:31:47 -07:00
elif self . status == " Skipped " :
2017-04-07 06:15:29 -07:00
color = GREY
glyph = CIRCLE
2017-04-06 10:31:47 -07:00
2017-04-07 06:15:29 -07:00
return color [ 1 ] + " %s | %s %s | %s s \n " % ( self . name . ljust ( self . padding ) , glyph , self . status . ljust ( 7 ) , self . time ) + color [ 0 ]
2017-04-05 12:19:26 -07:00
2017-04-11 10:40:54 -07:00
@property
def was_successful ( self ) :
return self . status != " Failed "
2017-04-05 12:19:26 -07:00
2017-03-27 08:33:00 -07:00
def check_script_list ( src_dir ) :
""" Check scripts directory.
Check that there are no scripts in the functional tests directory which are
not being run by pull - tester . py . """
script_dir = src_dir + ' /test/functional/ '
python_files = set ( [ t for t in os . listdir ( script_dir ) if t [ - 3 : ] == " .py " ] )
missed_tests = list ( python_files - set ( map ( lambda x : x . split ( ) [ 0 ] , ALL_SCRIPTS + NON_SCRIPTS ) ) )
if len ( missed_tests ) != 0 :
2017-04-12 07:41:13 -07:00
print ( " %s WARNING! %s The following scripts are not being run: %s . Check the test lists in test_runner.py. " % ( BOLD [ 1 ] , BOLD [ 0 ] , str ( missed_tests ) ) )
if os . getenv ( ' TRAVIS ' ) == ' true ' :
# On travis this warning is an error to prevent merging incomplete commits into master
sys . exit ( 1 )
2015-10-10 22:41:19 -07:00
class RPCCoverage ( object ) :
"""
2017-03-09 06:53:26 -08:00
Coverage reporting utilities for test_runner .
2015-10-10 22:41:19 -07:00
Coverage calculation works by having each test script subprocess write
coverage files into a particular directory . These files contain the RPC
commands invoked during testing , as well as a complete listing of RPC
commands per ` bitcoin - cli help ` ( ` rpc_interface . txt ` ) .
After all tests complete , the commands run are combined and diff ' d against
the complete list to calculate uncovered RPC commands .
2017-03-09 06:44:57 -08:00
See also : test / functional / test_framework / coverage . py
2015-10-10 22:41:19 -07:00
"""
def __init__ ( self ) :
self . dir = tempfile . mkdtemp ( prefix = " coverage " )
2016-04-29 03:51:15 -07:00
self . flag = ' --coveragedir= %s ' % self . dir
2015-10-10 22:41:19 -07:00
def report_rpc_coverage ( self ) :
"""
Print out RPC commands that were unexercised by tests .
"""
uncovered = self . _get_uncovered_rpc_commands ( )
if uncovered :
print ( " Uncovered RPC commands: " )
print ( " " . join ( ( " - %s \n " % i ) for i in sorted ( uncovered ) ) )
else :
print ( " All RPC commands covered. " )
def cleanup ( self ) :
return shutil . rmtree ( self . dir )
def _get_uncovered_rpc_commands ( self ) :
"""
Return a set of currently untested RPC commands .
"""
2017-03-09 06:44:57 -08:00
# This is shared from `test/functional/test-framework/coverage.py`
2017-02-06 06:07:14 -08:00
reference_filename = ' rpc_interface.txt '
coverage_file_prefix = ' coverage. '
2015-10-10 22:41:19 -07:00
2017-02-06 06:07:14 -08:00
coverage_ref_filename = os . path . join ( self . dir , reference_filename )
2015-10-10 22:41:19 -07:00
coverage_filenames = set ( )
all_cmds = set ( )
covered_cmds = set ( )
if not os . path . isfile ( coverage_ref_filename ) :
raise RuntimeError ( " No coverage reference found " )
with open ( coverage_ref_filename , ' r ' ) as f :
all_cmds . update ( [ i . strip ( ) for i in f . readlines ( ) ] )
for root , dirs , files in os . walk ( self . dir ) :
for filename in files :
2017-02-06 06:07:14 -08:00
if filename . startswith ( coverage_file_prefix ) :
2015-10-10 22:41:19 -07:00
coverage_filenames . add ( os . path . join ( root , filename ) )
for filename in coverage_filenames :
with open ( filename , ' r ' ) as f :
covered_cmds . update ( [ i . strip ( ) for i in f . readlines ( ) ] )
return all_cmds - covered_cmds
if __name__ == ' __main__ ' :
2017-02-06 06:07:14 -08:00
main ( )