Add logging to bitcoin-util-test.py

- Use the python standard logging library
- Run all tests and report all failing test-cases (rather than stop after one test case fails)
- If output is different from expected output, log a contextual diff.
This commit is contained in:
jnewbery 2016-10-24 12:49:25 +01:00
parent a4fd8dff68
commit 32c0d6e1d2
2 changed files with 54 additions and 24 deletions

View File

@ -7,6 +7,8 @@ import os
import json import json
import sys import sys
import binascii import binascii
import difflib
import logging
def parse_output(a, fmt): def parse_output(a, fmt):
if fmt == 'json': # json: compare parsed data if fmt == 'json': # json: compare parsed data
@ -33,53 +35,70 @@ def bctest(testDir, testObj, exeext):
if "output_cmp" in testObj: if "output_cmp" in testObj:
outputFn = testObj['output_cmp'] outputFn = testObj['output_cmp']
outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
try:
outputData = open(testDir + "/" + outputFn).read() outputData = open(testDir + "/" + outputFn).read()
except:
logging.error("Output file " + outputFn + " can not be opened")
raise
if not outputData: if not outputData:
print("Output data missing for " + outputFn) logging.error("Output data missing for " + outputFn)
sys.exit(1) raise Exception
proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True) proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
try: try:
outs = proc.communicate(input=inputData) outs = proc.communicate(input=inputData)
except OSError: except OSError:
print("OSError, Failed to execute " + execprog) logging.error("OSError, Failed to execute " + execprog)
sys.exit(1) raise
if outputData: if outputData:
try: try:
a_parsed = parse_output(outs[0], outputType) a_parsed = parse_output(outs[0], outputType)
except Exception as e: except Exception as e:
print('Error parsing command output as %s: %s' % (outputType,e)) logging.error('Error parsing command output as %s: %s' % (outputType,e))
sys.exit(1) raise
try: try:
b_parsed = parse_output(outputData, outputType) b_parsed = parse_output(outputData, outputType)
except Exception as e: except Exception as e:
print('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e)) logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
sys.exit(1) raise
if a_parsed != b_parsed: if a_parsed != b_parsed:
print("Output data mismatch for " + outputFn + " (format " + outputType + ")") logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
sys.exit(1) raise Exception
if outs[0] != outputData: if outs[0] != outputData:
print("Output formatting mismatch for " + outputFn + " (format " + outputType + ")") error_message = "Output formatting mismatch for " + outputFn + ":\n"
sys.exit(1) error_message += "".join(difflib.context_diff(outputData.splitlines(True),
outs[0].splitlines(True),
fromfile=outputFn,
tofile="returned"))
logging.error(error_message)
raise Exception
wantRC = 0 wantRC = 0
if "return_code" in testObj: if "return_code" in testObj:
wantRC = testObj['return_code'] wantRC = testObj['return_code']
if proc.returncode != wantRC: if proc.returncode != wantRC:
print("Return code mismatch for " + outputFn) logging.error("Return code mismatch for " + outputFn)
sys.exit(1) raise Exception
def bctester(testDir, input_basename, buildenv, verbose = False): def bctester(testDir, input_basename, buildenv):
input_filename = testDir + "/" + input_basename input_filename = testDir + "/" + input_basename
raw_data = open(input_filename).read() raw_data = open(input_filename).read()
input_data = json.loads(raw_data) input_data = json.loads(raw_data)
for testObj in input_data: failed_testcases = []
if verbose and "description" in testObj:
print ("Testing: " + testObj["description"])
bctest(testDir, testObj, buildenv.exeext)
if verbose and "description" in testObj:
print ("PASS")
for testObj in input_data:
try:
bctest(testDir, testObj, buildenv.exeext)
logging.info("PASSED: " + testObj["description"])
except:
logging.info("FAILED: " + testObj["description"])
failed_testcases.append(testObj["description"])
if failed_testcases:
logging.error("FAILED TESTCASES: [" + ", ".join(failed_testcases) + "]")
sys.exit(1)
else:
sys.exit(0) sys.exit(0)

View File

@ -4,9 +4,11 @@
# file COPYING or http://www.opensource.org/licenses/mit-license.php. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
from __future__ import division,print_function,unicode_literals from __future__ import division,print_function,unicode_literals
import os import os
import sys
import bctest import bctest
import buildenv import buildenv
import argparse import argparse
import logging
help_text="""Test framework for bitcoin utils. help_text="""Test framework for bitcoin utils.
@ -19,9 +21,9 @@ test/bitcoin-util-test.py --src=[srcdir]
if __name__ == '__main__': if __name__ == '__main__':
verbose = False
try: try:
srcdir = os.environ["srcdir"] srcdir = os.environ["srcdir"]
verbose = False
except: except:
parser = argparse.ArgumentParser(description=help_text) parser = argparse.ArgumentParser(description=help_text)
parser.add_argument('-s', '--srcdir') parser.add_argument('-s', '--srcdir')
@ -29,4 +31,13 @@ if __name__ == '__main__':
args = parser.parse_args() args = parser.parse_args()
srcdir = args.srcdir srcdir = args.srcdir
verbose = args.verbose verbose = args.verbose
bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv, verbose = verbose)
if verbose:
level = logging.DEBUG
else:
level = logging.ERROR
formatter = '%(asctime)s - %(levelname)s - %(message)s'
# Add the format/level to the logger
logging.basicConfig(format = formatter, level=level)
bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv)