included-tests: generate binary data from test files for inclusion into test binaries

This change moves test data into the binaries rather than reading them from
the disk at runtime.

Advantages:
- Tests become distributable
- Cross-compile friendly. Build on one machine and execute in an arbitrary
  location on another.
- Easier testing for backports. Users can verify that tests pass without having
  to track down corresponding test data.
- More trustworthy test results and easier quality assurance as tests make
  fewer assumptions about their environment.
- Tests could theoretically run at client/daemon startup and exit on failure.

Disadvantages:
- Required 'hexdump' build-dependency. This is a standard bsd tool that should
  be usable everywhere. It is likely already installed on all build-machines.
- Tests can no longer be fudged after build by altering test-data.
This commit is contained in:
Cory Fields 2013-09-10 15:18:09 -04:00
parent 08081e393b
commit 152e51c7af
11 changed files with 76 additions and 72 deletions

3
.gitignore vendored
View File

@ -57,6 +57,9 @@ src/qt/res/.dirstamp
*.trs *.trs
*.dmg *.dmg
*.json.h
*.raw.h
# Compilation and Qt preprocessor part # Compilation and Qt preprocessor part
*.qm *.qm
Makefile Makefile

View File

@ -151,6 +151,7 @@ AC_PATH_PROGS([LUPDATE], [lupdate-qt4 lupdate4 lupdate],, $qt_bin_path:$PATH)
AC_PATH_PROG([PROTOC], [protoc],, $protoc_bin_path:$PATH) AC_PATH_PROG([PROTOC], [protoc],, $protoc_bin_path:$PATH)
AC_PATH_PROG(CCACHE,ccache) AC_PATH_PROG(CCACHE,ccache)
AC_PATH_PROG(XGETTEXT,xgettext) AC_PATH_PROG(XGETTEXT,xgettext)
AC_PATH_PROG(HEXDUMP,hexdump)
PKG_PROG_PKG_CONFIG PKG_PROG_PKG_CONFIG
## TODO: Remove these hard-coded paths and flags. They are here for the sake of ## TODO: Remove these hard-coded paths and flags. They are here for the sake of
@ -353,6 +354,12 @@ AX_BOOST_THREAD
AX_BOOST_CHRONO AX_BOOST_CHRONO
if test x$use_tests = xyes; then if test x$use_tests = xyes; then
if test x$HEXDUMP = x; then
AC_MSG_ERROR(hexdump is required for tests)
fi
AX_BOOST_UNIT_TEST_FRAMEWORK AX_BOOST_UNIT_TEST_FRAMEWORK
fi fi

View File

@ -1,6 +1,6 @@
AM_CPPFLAGS = $(INCLUDES) -I$(top_builddir)/src/obj \ AM_CPPFLAGS = $(INCLUDES) -I$(top_builddir)/src/obj \
-I$(top_srcdir)/src/leveldb/include -I$(top_srcdir)/src/leveldb/helpers \ -I$(top_srcdir)/src/leveldb/include -I$(top_srcdir)/src/leveldb/helpers \
$(BOOST_INCLUDES) -I$(builddir) $(BOOST_INCLUDES)
AM_LDFLAGS = $(PTHREAD_CFLAGS) AM_LDFLAGS = $(PTHREAD_CFLAGS)
noinst_LIBRARIES = libbitcoin.a noinst_LIBRARIES = libbitcoin.a

View File

@ -49,3 +49,19 @@ moc_%.cpp: %.h
%.pb.cc %.pb.h: %.proto %.pb.cc %.pb.h: %.proto
test -f $(PROTOC) && $(PROTOC) --cpp_out=$(@D) --proto_path=$(abspath $(<D) $<) || \ test -f $(PROTOC) && $(PROTOC) --cpp_out=$(@D) --proto_path=$(abspath $(<D) $<) || \
echo error: could not build $@ echo error: could not build $@
%.json.h: %.json
@$(MKDIR_P) $(@D)
@echo "namespace json_tests{" > $@
@echo "static unsigned const char $(*F)[] = {" >> $@
@$(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' >> $@
@echo "};};" >> $@
@echo "Generated $@"
%.raw.h: %.raw
@$(MKDIR_P) $(@D)
@echo "namespace alert_tests{" > $@
@echo "static unsigned const char $(*F)[] = {" >> $@
@$(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' >> $@
@echo "};};" >> $@
@echo "Generated $@"

View File

@ -10,19 +10,20 @@ bin_PROGRAMS = test_bitcoin
TESTS = test_bitcoin TESTS = test_bitcoin
TEST_DATA_DIR=$(srcdir)/data JSON_TEST_FILES= data/script_valid.json \
data/base58_keys_valid.json data/sig_canonical.json \
data/sig_noncanonical.json \
data/base58_encode_decode.json \
data/base58_keys_invalid.json \
data/script_invalid.json data/tx_invalid.json \
data/tx_valid.json
TEST_DATA_FILES= $(TEST_DATA_DIR)/script_valid.json \ RAW_TEST_FILES = data/alertTests.raw
$(TEST_DATA_DIR)/base58_keys_valid.json $(TEST_DATA_DIR)/sig_canonical.json \
$(TEST_DATA_DIR)/sig_noncanonical.json \ BUILT_SOURCES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h)
$(TEST_DATA_DIR)/base58_encode_decode.json $(TEST_DATA_DIR)/alertTests \
$(TEST_DATA_DIR)/base58_keys_invalid.json \
$(TEST_DATA_DIR)/script_invalid.json $(TEST_DATA_DIR)/tx_invalid.json \
$(TEST_DATA_DIR)/tx_valid.json
# test_bitcoin binary # # test_bitcoin binary #
test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(TESTDEFS) \ test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(TESTDEFS)
-DTEST_DATA_DIR=$(abs_srcdir)/data
test_bitcoin_LDADD = $(LIBBITCOIN) $(LIBLEVELDB) $(LIBMEMENV) \ test_bitcoin_LDADD = $(LIBBITCOIN) $(LIBLEVELDB) $(LIBMEMENV) \
$(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB)
test_bitcoin_SOURCES = accounting_tests.cpp alert_tests.cpp \ test_bitcoin_SOURCES = accounting_tests.cpp alert_tests.cpp \
@ -33,6 +34,8 @@ test_bitcoin_SOURCES = accounting_tests.cpp alert_tests.cpp \
netbase_tests.cpp pmt_tests.cpp rpc_tests.cpp script_P2SH_tests.cpp \ netbase_tests.cpp pmt_tests.cpp rpc_tests.cpp script_P2SH_tests.cpp \
script_tests.cpp serialize_tests.cpp sigopcount_tests.cpp test_bitcoin.cpp \ script_tests.cpp serialize_tests.cpp sigopcount_tests.cpp test_bitcoin.cpp \
transaction_tests.cpp uint160_tests.cpp uint256_tests.cpp util_tests.cpp \ transaction_tests.cpp uint160_tests.cpp uint256_tests.cpp util_tests.cpp \
wallet_tests.cpp $(TEST_DATA_FILES) wallet_tests.cpp $(JSON_TEST_FILES) $(RAW_TEST_FILES)
CLEANFILES = *.gcda *.gcno nodist_test_bitcoin_SOURCES = $(BUILT_SOURCES)
CLEANFILES = *.gcda *.gcno $(BUILT_SOURCES)

View File

@ -9,6 +9,7 @@
#include "alert.h" #include "alert.h"
#include "serialize.h" #include "serialize.h"
#include "util.h" #include "util.h"
#include "data/alertTests.raw.h"
#if 0 #if 0
// //
@ -71,27 +72,13 @@ struct ReadAlerts
{ {
ReadAlerts() ReadAlerts()
{ {
std::string filename("alertTests"); std::vector<unsigned char> vch(alert_tests::alertTests, alert_tests::alertTests + sizeof(alert_tests::alertTests));
namespace fs = boost::filesystem; CDataStream stream(vch, SER_DISK, CLIENT_VERSION);
fs::path testFile = fs::current_path() / "data" / filename;
#ifdef TEST_DATA_DIR
if (!fs::exists(testFile))
{
testFile = fs::path(BOOST_PP_STRINGIZE(TEST_DATA_DIR)) / filename;
}
#endif
FILE* fp = fopen(testFile.string().c_str(), "rb");
if (!fp) return;
CAutoFile filein = CAutoFile(fp, SER_DISK, CLIENT_VERSION);
if (!filein) return;
try { try {
while (!feof(filein)) while (stream.good())
{ {
CAlert alert; CAlert alert;
filein >> alert; stream >> alert;
alerts.push_back(alert); alerts.push_back(alert);
} }
} }

View File

@ -2,20 +2,21 @@
#include "json/json_spirit_reader_template.h" #include "json/json_spirit_reader_template.h"
#include "json/json_spirit_writer_template.h" #include "json/json_spirit_writer_template.h"
#include "json/json_spirit_utils.h" #include "json/json_spirit_utils.h"
#include "data/base58_encode_decode.json.h"
#include "data/base58_keys_invalid.json.h"
#include "data/base58_keys_valid.json.h"
#include "base58.h" #include "base58.h"
#include "util.h" #include "util.h"
using namespace json_spirit; using namespace json_spirit;
extern Array read_json(const std::string& filename); extern Array read_json(const std::string& jsondata);
BOOST_AUTO_TEST_SUITE(base58_tests) BOOST_AUTO_TEST_SUITE(base58_tests)
// Goal: test low-level base58 encoding functionality // Goal: test low-level base58 encoding functionality
BOOST_AUTO_TEST_CASE(base58_EncodeBase58) BOOST_AUTO_TEST_CASE(base58_EncodeBase58)
{ {
Array tests = read_json("base58_encode_decode.json"); Array tests = read_json(std::string(json_tests::base58_encode_decode, json_tests::base58_encode_decode + sizeof(json_tests::base58_encode_decode)));
BOOST_FOREACH(Value& tv, tests) BOOST_FOREACH(Value& tv, tests)
{ {
Array test = tv.get_array(); Array test = tv.get_array();
@ -36,7 +37,7 @@ BOOST_AUTO_TEST_CASE(base58_EncodeBase58)
// Goal: test low-level base58 decoding functionality // Goal: test low-level base58 decoding functionality
BOOST_AUTO_TEST_CASE(base58_DecodeBase58) BOOST_AUTO_TEST_CASE(base58_DecodeBase58)
{ {
Array tests = read_json("base58_encode_decode.json"); Array tests = read_json(std::string(json_tests::base58_encode_decode, json_tests::base58_encode_decode + sizeof(json_tests::base58_encode_decode)));
std::vector<unsigned char> result; std::vector<unsigned char> result;
BOOST_FOREACH(Value& tv, tests) BOOST_FOREACH(Value& tv, tests)
@ -104,7 +105,7 @@ public:
// Goal: check that parsed keys match test payload // Goal: check that parsed keys match test payload
BOOST_AUTO_TEST_CASE(base58_keys_valid_parse) BOOST_AUTO_TEST_CASE(base58_keys_valid_parse)
{ {
Array tests = read_json("base58_keys_valid.json"); Array tests = read_json(std::string(json_tests::base58_keys_valid, json_tests::base58_keys_valid + sizeof(json_tests::base58_keys_valid)));
std::vector<unsigned char> result; std::vector<unsigned char> result;
CBitcoinSecret secret; CBitcoinSecret secret;
CBitcoinAddress addr; CBitcoinAddress addr;
@ -163,7 +164,7 @@ BOOST_AUTO_TEST_CASE(base58_keys_valid_parse)
// Goal: check that generated keys match test vectors // Goal: check that generated keys match test vectors
BOOST_AUTO_TEST_CASE(base58_keys_valid_gen) BOOST_AUTO_TEST_CASE(base58_keys_valid_gen)
{ {
Array tests = read_json("base58_keys_valid.json"); Array tests = read_json(std::string(json_tests::base58_keys_valid, json_tests::base58_keys_valid + sizeof(json_tests::base58_keys_valid)));
std::vector<unsigned char> result; std::vector<unsigned char> result;
BOOST_FOREACH(Value& tv, tests) BOOST_FOREACH(Value& tv, tests)
{ {
@ -231,7 +232,7 @@ BOOST_AUTO_TEST_CASE(base58_keys_valid_gen)
// Goal: check that base58 parsing code is robust against a variety of corrupted data // Goal: check that base58 parsing code is robust against a variety of corrupted data
BOOST_AUTO_TEST_CASE(base58_keys_invalid) BOOST_AUTO_TEST_CASE(base58_keys_invalid)
{ {
Array tests = read_json("base58_keys_invalid.json"); // Negative testcases Array tests = read_json(std::string(json_tests::base58_keys_invalid, json_tests::base58_keys_invalid + sizeof(json_tests::base58_keys_invalid))); // Negative testcases
std::vector<unsigned char> result; std::vector<unsigned char> result;
CBitcoinSecret secret; CBitcoinSecret secret;
CBitcoinAddress addr; CBitcoinAddress addr;

View File

@ -8,13 +8,15 @@
#include "key.h" #include "key.h"
#include "script.h" #include "script.h"
#include "util.h" #include "util.h"
#include "data/sig_noncanonical.json.h"
#include "data/sig_canonical.json.h"
using namespace std; using namespace std;
using namespace json_spirit; using namespace json_spirit;
// In script_tests.cpp // In script_tests.cpp
extern Array read_json(const std::string& filename); extern Array read_json(const std::string& jsondata);
BOOST_AUTO_TEST_SUITE(canonical_tests) BOOST_AUTO_TEST_SUITE(canonical_tests)
@ -58,7 +60,7 @@ bool static IsCanonicalSignature_OpenSSL(const std::vector<unsigned char> &vchSi
BOOST_AUTO_TEST_CASE(script_canon) BOOST_AUTO_TEST_CASE(script_canon)
{ {
Array tests = read_json("sig_canonical.json"); Array tests = read_json(std::string(json_tests::sig_canonical, json_tests::sig_canonical + sizeof(json_tests::sig_canonical)));
BOOST_FOREACH(Value &tv, tests) { BOOST_FOREACH(Value &tv, tests) {
string test = tv.get_str(); string test = tv.get_str();
@ -72,7 +74,7 @@ BOOST_AUTO_TEST_CASE(script_canon)
BOOST_AUTO_TEST_CASE(script_noncanon) BOOST_AUTO_TEST_CASE(script_noncanon)
{ {
Array tests = read_json("sig_noncanonical.json"); Array tests = read_json(std::string(json_tests::sig_noncanonical, json_tests::sig_noncanonical + sizeof(json_tests::sig_noncanonical)));
BOOST_FOREACH(Value &tv, tests) { BOOST_FOREACH(Value &tv, tests) {
string test = tv.get_str(); string test = tv.get_str();

View File

@ -14,6 +14,8 @@
#include "main.h" #include "main.h"
#include "wallet.h" #include "wallet.h"
#include "data/script_invalid.json.h"
#include "data/script_valid.json.h"
using namespace std; using namespace std;
using namespace json_spirit; using namespace json_spirit;
@ -90,34 +92,15 @@ ParseScript(string s)
} }
Array Array
read_json(const std::string& filename) read_json(const std::string& jsondata)
{ {
namespace fs = boost::filesystem;
fs::path testFile = fs::current_path() / "data" / filename;
#ifdef TEST_DATA_DIR
if (!fs::exists(testFile))
{
testFile = fs::path(BOOST_PP_STRINGIZE(TEST_DATA_DIR)) / filename;
}
#endif
ifstream ifs(testFile.string().c_str(), ifstream::in);
Value v; Value v;
if (!read_stream(ifs, v))
{
if (ifs.fail())
BOOST_ERROR("Cound not find/open " << filename);
else
BOOST_ERROR("JSON syntax error in " << filename);
return Array();
}
if (v.type() != array_type)
{
BOOST_ERROR(filename << " does not contain a json array");
return Array();
}
if (!read_string(jsondata, v) || v.type() != array_type)
{
BOOST_ERROR("Parse error.");
return Array();
}
return v.get_array(); return v.get_array();
} }
@ -130,7 +113,7 @@ BOOST_AUTO_TEST_CASE(script_valid)
// Inner arrays are [ "scriptSig", "scriptPubKey" ] // Inner arrays are [ "scriptSig", "scriptPubKey" ]
// ... where scriptSig and scriptPubKey are stringified // ... where scriptSig and scriptPubKey are stringified
// scripts. // scripts.
Array tests = read_json("script_valid.json"); Array tests = read_json(std::string(json_tests::script_valid, json_tests::script_valid + sizeof(json_tests::script_valid)));
BOOST_FOREACH(Value& tv, tests) BOOST_FOREACH(Value& tv, tests)
{ {
@ -154,7 +137,7 @@ BOOST_AUTO_TEST_CASE(script_valid)
BOOST_AUTO_TEST_CASE(script_invalid) BOOST_AUTO_TEST_CASE(script_invalid)
{ {
// Scripts that should evaluate as invalid // Scripts that should evaluate as invalid
Array tests = read_json("script_invalid.json"); Array tests = read_json(std::string(json_tests::script_invalid, json_tests::script_invalid + sizeof(json_tests::script_invalid)));
BOOST_FOREACH(Value& tv, tests) BOOST_FOREACH(Value& tv, tests)
{ {

View File

@ -5,12 +5,14 @@
#include "main.h" #include "main.h"
#include "wallet.h" #include "wallet.h"
#include "data/tx_invalid.json.h"
#include "data/tx_valid.json.h"
using namespace std; using namespace std;
using namespace json_spirit; using namespace json_spirit;
// In script_tests.cpp // In script_tests.cpp
extern Array read_json(const std::string& filename); extern Array read_json(const std::string& jsondata);
extern CScript ParseScript(string s); extern CScript ParseScript(string s);
BOOST_AUTO_TEST_SUITE(transaction_tests) BOOST_AUTO_TEST_SUITE(transaction_tests)
@ -22,7 +24,7 @@ BOOST_AUTO_TEST_CASE(tx_valid)
// Inner arrays are either [ "comment" ] // Inner arrays are either [ "comment" ]
// or [[[prevout hash, prevout index, prevout scriptPubKey], [input 2], ...],"], serializedTransaction, enforceP2SH // or [[[prevout hash, prevout index, prevout scriptPubKey], [input 2], ...],"], serializedTransaction, enforceP2SH
// ... where all scripts are stringified scripts. // ... where all scripts are stringified scripts.
Array tests = read_json("tx_valid.json"); Array tests = read_json(std::string(json_tests::tx_valid, json_tests::tx_valid + sizeof(json_tests::tx_valid)));
BOOST_FOREACH(Value& tv, tests) BOOST_FOREACH(Value& tv, tests)
{ {
@ -91,7 +93,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid)
// Inner arrays are either [ "comment" ] // Inner arrays are either [ "comment" ]
// or [[[prevout hash, prevout index, prevout scriptPubKey], [input 2], ...],"], serializedTransaction, enforceP2SH // or [[[prevout hash, prevout index, prevout scriptPubKey], [input 2], ...],"], serializedTransaction, enforceP2SH
// ... where all scripts are stringified scripts. // ... where all scripts are stringified scripts.
Array tests = read_json("tx_invalid.json"); Array tests = read_json(std::string(json_tests::tx_invalid, json_tests::tx_invalid + sizeof(json_tests::tx_invalid)));
BOOST_FOREACH(Value& tv, tests) BOOST_FOREACH(Value& tv, tests)
{ {