diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 146a8ddb5..603b6d9ed 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -3,16 +3,32 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Run Regression Test Suite -# +""" +Run Regression Test Suite + +This module calls down into individual test cases via subprocess. It will +forward all unrecognized arguments onto the individual test scripts, other +than: + + - `-extended`: run the "extended" test suite in addition to the basic one. + - `-win`: signal that this is running in a Windows environment, and we + should run the tests. + - `--coverage`: this generates a basic coverage report for the RPC + interface. + +For a description of arguments recognized by test scripts, see +`qa/pull-tester/test_framework/test_framework.py:BitcoinTestFramework.main`. + +""" import os +import shutil import sys import subprocess +import tempfile import re + from tests_config import * -from sets import Set #If imported values are not defined then set to zero (or disabled) if not vars().has_key('ENABLE_WALLET'): @@ -24,15 +40,20 @@ if not vars().has_key('ENABLE_UTILS'): if not vars().has_key('ENABLE_ZMQ'): ENABLE_ZMQ=0 +ENABLE_COVERAGE=0 + #Create a set to store arguments and create the passOn string -opts = Set() +opts = set() passOn = "" p = re.compile("^--") -for i in range(1,len(sys.argv)): - if (p.match(sys.argv[i]) or sys.argv[i] == "-h"): - passOn += " " + sys.argv[i] + +for arg in sys.argv[1:]: + if arg == '--coverage': + ENABLE_COVERAGE = 1 + elif (p.match(arg) or arg == "-h"): + passOn += " " + arg else: - opts.add(sys.argv[i]) + opts.add(arg) #Set env vars buildDir = BUILDDIR @@ -152,24 +173,125 @@ testScriptsExt = [ if ENABLE_ZMQ == 1: testScripts.append('zmq_test.py') -if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): - rpcTestDir = buildDir + '/qa/rpc-tests/' - #Run Tests - for i in range(len(testScripts)): - if (len(opts) == 0 or (len(opts) == 1 and "-win" in opts ) or '-extended' in opts - or testScripts[i] in opts or re.sub(".py$", "", testScripts[i]) in opts ): - print "Running testscript " + testScripts[i] + "..." - subprocess.check_call(rpcTestDir + testScripts[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) - #exit if help is called so we print just one set of instructions - p = re.compile(" -h| --help") - if p.match(passOn): - sys.exit(0) - #Run Extended Tests - for i in range(len(testScriptsExt)): - if ('-extended' in opts or testScriptsExt[i] in opts - or re.sub(".py$", "", testScriptsExt[i]) in opts): - print "Running 2nd level testscript " + testScriptsExt[i] + "..." - subprocess.check_call(rpcTestDir + testScriptsExt[i] + " --srcdir " + buildDir + '/src ' + passOn,shell=True) -else: - print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled" +def runtests(): + coverage = None + + if ENABLE_COVERAGE: + coverage = RPCCoverage() + print("Initializing coverage directory at %s" % coverage.dir) + + if(ENABLE_WALLET == 1 and ENABLE_UTILS == 1 and ENABLE_BITCOIND == 1): + rpcTestDir = buildDir + '/qa/rpc-tests/' + run_extended = '-extended' in opts + cov_flag = coverage.flag if coverage else '' + flags = " --srcdir %s/src %s %s" % (buildDir, cov_flag, passOn) + + #Run Tests + for i in range(len(testScripts)): + if (len(opts) == 0 + or (len(opts) == 1 and "-win" in opts ) + or run_extended + or testScripts[i] in opts + or re.sub(".py$", "", testScripts[i]) in opts ): + print("Running testscript " + testScripts[i] + "...") + + subprocess.check_call( + rpcTestDir + testScripts[i] + flags, shell=True) + + # exit if help is called so we print just one set of + # instructions + p = re.compile(" -h| --help") + if p.match(passOn): + sys.exit(0) + + # Run Extended Tests + for i in range(len(testScriptsExt)): + if (run_extended or testScriptsExt[i] in opts + or re.sub(".py$", "", testScriptsExt[i]) in opts): + print( + "Running 2nd level testscript " + + testScriptsExt[i] + "...") + + subprocess.check_call( + rpcTestDir + testScriptsExt[i] + flags, shell=True) + + if coverage: + coverage.report_rpc_coverage() + + print("Cleaning up coverage data") + coverage.cleanup() + + else: + print "No rpc tests to run. Wallet, utils, and bitcoind must all be enabled" + + +class RPCCoverage(object): + """ + Coverage reporting utilities for pull-tester. + + Coverage calculation works by having each test script subprocess write + coverage files into a particular directory. These files contain the RPC + commands invoked during testing, as well as a complete listing of RPC + commands per `bitcoin-cli help` (`rpc_interface.txt`). + + After all tests complete, the commands run are combined and diff'd against + the complete list to calculate uncovered RPC commands. + + See also: qa/rpc-tests/test_framework/coverage.py + + """ + def __init__(self): + self.dir = tempfile.mkdtemp(prefix="coverage") + self.flag = '--coveragedir %s' % self.dir + + def report_rpc_coverage(self): + """ + Print out RPC commands that were unexercised by tests. + + """ + uncovered = self._get_uncovered_rpc_commands() + + if uncovered: + print("Uncovered RPC commands:") + print("".join((" - %s\n" % i) for i in sorted(uncovered))) + else: + print("All RPC commands covered.") + + def cleanup(self): + return shutil.rmtree(self.dir) + + def _get_uncovered_rpc_commands(self): + """ + Return a set of currently untested RPC commands. + + """ + # This is shared from `qa/rpc-tests/test-framework/coverage.py` + REFERENCE_FILENAME = 'rpc_interface.txt' + COVERAGE_FILE_PREFIX = 'coverage.' + + coverage_ref_filename = os.path.join(self.dir, REFERENCE_FILENAME) + coverage_filenames = set() + all_cmds = set() + covered_cmds = set() + + if not os.path.isfile(coverage_ref_filename): + raise RuntimeError("No coverage reference found") + + with open(coverage_ref_filename, 'r') as f: + all_cmds.update([i.strip() for i in f.readlines()]) + + for root, dirs, files in os.walk(self.dir): + for filename in files: + if filename.startswith(COVERAGE_FILE_PREFIX): + coverage_filenames.add(os.path.join(root, filename)) + + for filename in coverage_filenames: + with open(filename, 'r') as f: + covered_cmds.update([i.strip() for i in f.readlines()]) + + return all_cmds - covered_cmds + + +if __name__ == '__main__': + runtests()