From bec25c91cb2c15140431583aa476fc29eca54a71 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 22 Feb 2017 13:18:37 +0000 Subject: [PATCH 01/41] Revert "Closes #1680, temporary fix for rpc deadlock inherited from upstream." This reverts commit f4404d7b5bed29972d7977ef60f305842a740c8a. --- qa/rpc-tests/httpbasics.py | 21 +++++++++++++-------- src/init.cpp | 5 +---- src/rpcserver.cpp | 6 ------ 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/qa/rpc-tests/httpbasics.py b/qa/rpc-tests/httpbasics.py index 20f3218d7..8ccb82128 100755 --- a/qa/rpc-tests/httpbasics.py +++ b/qa/rpc-tests/httpbasics.py @@ -38,9 +38,13 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read(); assert_equal('"error":null' in out1, True) + assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open! - # TODO #1856: Re-enable support for persistent connections. - assert_equal(conn.sock!=None, False) + #send 2nd request without closing connection + conn.request('POST', '/', '{"method": "getchaintips"}', headers) + out2 = conn.getresponse().read(); + assert_equal('"error":null' in out1, True) #must also response with a correct json-rpc message + assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open! conn.close() #same should be if we add keep-alive because this should be the std. behaviour @@ -51,9 +55,13 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read(); assert_equal('"error":null' in out1, True) + assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open! - # TODO #1856: Re-enable support for persistent connections. - assert_equal(conn.sock!=None, False) + #send 2nd request without closing connection + conn.request('POST', '/', '{"method": "getchaintips"}', headers) + out2 = conn.getresponse().read(); + assert_equal('"error":null' in out1, True) #must also response with a correct json-rpc message + assert_equal(conn.sock!=None, True) #according to http/1.1 connection must still be open! conn.close() #now do the same with "Connection: close" @@ -88,10 +96,7 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read(); assert_equal('"error":null' in out1, True) - - # TODO #1856: Re-enable support for persistent connections. - assert_equal(conn.sock!=None, False) - conn.close() + assert_equal(conn.sock!=None, True) #connection must be closed because bitcoind should use keep-alive by default if __name__ == '__main__': HTTPBasicsTest ().main () diff --git a/src/init.cpp b/src/init.cpp index aeb877d4c..0a35b2828 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -467,10 +467,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-rpcport=", strprintf(_("Listen for JSON-RPC connections on (default: %u or testnet: %u)"), 8232, 18232)); strUsage += HelpMessageOpt("-rpcallowip=", _("Allow JSON-RPC connections from specified source. Valid for are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times")); strUsage += HelpMessageOpt("-rpcthreads=", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), 4)); - - // TODO #1856: Re-enable support for persistent connections. - // Disabled to avoid rpc deadlock #1680, until we backport upstream changes which replace boost::asio with libevent, or another solution is implemented. - //strUsage += HelpMessageOpt("-rpckeepalive", strprintf(_("RPC support for HTTP persistent connections (default: %d)"), 1)); + strUsage += HelpMessageOpt("-rpckeepalive", strprintf(_("RPC support for HTTP persistent connections (default: %d)"), 1)); // Disabled until we can lock notes and also tune performance of libsnark which by default uses multiple threads //strUsage += HelpMessageOpt("-rpcasyncthreads=", strprintf(_("Set the number of threads to service Async RPC calls (default: %d)"), 1)); diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index c8f383f0e..6ede4e227 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -1023,15 +1023,9 @@ void ServiceConnection(AcceptedConnection *conn) // Read HTTP message headers and body ReadHTTPMessage(conn->stream(), mapHeaders, strRequest, nProto, MAX_SIZE); - // TODO #1856: Re-enable support for persistent connections. - // We have disabled support for HTTP Keep-Alive until resolution of #1680, upstream rpc deadlock. - // Close connection immediately. - fRun = false; - /* // HTTP Keep-Alive is false; close connection immediately if ((mapHeaders["connection"] == "close") || (!GetBoolArg("-rpckeepalive", true))) fRun = false; - */ // Process via JSON-RPC API if (strURI == "/") { From b6fbda09820caf00ceb18aaadfa91894256a81bf Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 23 Jan 2015 07:53:50 +0100 Subject: [PATCH 02/41] doc: remove documentation for rpcssl --- contrib/bitcoind.bash-completion | 2 +- contrib/debian/examples/zcash.conf | 9 --------- doc/man/zcash-cli.1 | 6 ------ doc/man/zcashd.1 | 19 ------------------- 4 files changed, 1 insertion(+), 35 deletions(-) diff --git a/contrib/bitcoind.bash-completion b/contrib/bitcoind.bash-completion index ea32ec3c4..104365024 100644 --- a/contrib/bitcoind.bash-completion +++ b/contrib/bitcoind.bash-completion @@ -16,7 +16,7 @@ _zcashd() { _get_comp_words_by_ref -n = cur prev words cword case "$cur" in - -conf=*|-pid=*|-loadblock=*|-rpccookiefile=*|-wallet=*|-rpcsslcertificatechainfile=*|-rpcsslprivatekeyfile=*) + -conf=*|-pid=*|-loadblock=*|-rpccookiefile=*|-wallet=*) cur="${cur#*=}" _filedir return 0 diff --git a/contrib/debian/examples/zcash.conf b/contrib/debian/examples/zcash.conf index cbfdf4ee7..cbe200496 100644 --- a/contrib/debian/examples/zcash.conf +++ b/contrib/debian/examples/zcash.conf @@ -95,15 +95,6 @@ # running on another host using this option: #rpcconnect=127.0.0.1 -# Use Secure Sockets Layer (also known as TLS or HTTPS) to communicate -# with Zcash -server or zcashd -#rpcssl=1 - -# OpenSSL settings used when rpcssl=1 -#rpcsslciphers=TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!AH:!3DES:@STRENGTH -#rpcsslcertificatechainfile=server.cert -#rpcsslprivatekeyfile=server.pem - # Transaction Fee # Send transactions as zero-fee transactions if possible (default: 0) diff --git a/doc/man/zcash-cli.1 b/doc/man/zcash-cli.1 index 1247efd4d..582861f7b 100644 --- a/doc/man/zcash-cli.1 +++ b/doc/man/zcash-cli.1 @@ -57,12 +57,6 @@ Username for JSON\-RPC connections \fB\-rpcpassword=\fR .IP Password for JSON\-RPC connections -.PP -SSL options: (see the Bitcoin Wiki for SSL setup instructions) -.HP -\fB\-rpcssl\fR -.IP -Use OpenSSL (https) for JSON\-RPC connections .SH COPYRIGHT Copyright (C) 2009-2017 The Bitcoin Core Developers Copyright (C) 2015-2017 The Zcash Developers diff --git a/doc/man/zcashd.1 b/doc/man/zcashd.1 index 9e0dae7ea..0a94615a0 100644 --- a/doc/man/zcashd.1 +++ b/doc/man/zcashd.1 @@ -422,25 +422,6 @@ multiple times .IP Set the number of threads to service RPC calls (default: 4) .PP -RPC SSL options: (see the Bitcoin Wiki for SSL setup instructions) -.HP -\fB\-rpcssl\fR -.IP -Use OpenSSL (https) for JSON\-RPC connections -.HP -\fB\-rpcsslcertificatechainfile=\fR -.IP -Server certificate file (default: server.cert) -.HP -\fB\-rpcsslprivatekeyfile=\fR -.IP -Server private key (default: server.pem) -.HP -\fB\-rpcsslciphers=\fR -.IP -Acceptable ciphers (default: -TLSv1.2+HIGH:TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!3DES:@STRENGTH) -.PP Metrics Options (only if \fB\-daemon\fR and \fB\-printtoconsole\fR are not set): .HP \fB\-showmetrics\fR From 14dcc6ab965a07d940059018fc06d982fc372ca1 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 23 Jan 2015 07:54:27 +0100 Subject: [PATCH 03/41] qa: Remove -rpckeepalive tests from httpbasics This option was a temporary workaround, and is no longer necessary with the new web server. --- qa/rpc-tests/httpbasics.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/httpbasics.py b/qa/rpc-tests/httpbasics.py index 8ccb82128..b66533543 100755 --- a/qa/rpc-tests/httpbasics.py +++ b/qa/rpc-tests/httpbasics.py @@ -22,7 +22,7 @@ except ImportError: class HTTPBasicsTest (BitcoinTestFramework): def setup_nodes(self): - return start_nodes(4, self.options.tmpdir, extra_args=[['-rpckeepalive=1'], ['-rpckeepalive=0'], [], []]) + return start_nodes(4, self.options.tmpdir) def run_test(self): @@ -84,9 +84,8 @@ class HTTPBasicsTest (BitcoinTestFramework): conn.request('POST', '/', '{"method": "getbestblockhash"}', headers) out1 = conn.getresponse().read(); assert_equal('"error":null' in out1, True) - assert_equal(conn.sock!=None, False) #connection must be closed because keep-alive was set to false - #node2 (third node) is running with standard keep-alive parameters which means keep-alive is off + #node2 (third node) is running with standard keep-alive parameters which means keep-alive is on urlNode2 = urlparse.urlparse(self.nodes[2].url) authpair = urlNode2.username + ':' + urlNode2.password headers = {"Authorization": "Basic " + base64.b64encode(authpair)} From 72f52420ec32a8dcb2d28c5637054ac721ab4cc0 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 23 Jan 2015 07:55:37 +0100 Subject: [PATCH 04/41] Remove rpc_boostasiotocnetaddr test Dropping all use of boost::asio. --- src/test/rpc_tests.cpp | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 30401c253..c2b29ab67 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -212,21 +212,6 @@ BOOST_AUTO_TEST_CASE(json_parse_errors) BOOST_CHECK_THROW(ParseNonRFCJSONValue("3J98t1WpEZ73CNmQviecrnyiWrnqRhWNL"), std::runtime_error); } -BOOST_AUTO_TEST_CASE(rpc_boostasiotocnetaddr) -{ - // Check IPv4 addresses - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("1.2.3.4")).ToString(), "1.2.3.4"); - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("127.0.0.1")).ToString(), "127.0.0.1"); - // Check IPv6 addresses - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("::1")).ToString(), "::1"); - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("123:4567:89ab:cdef:123:4567:89ab:cdef")).ToString(), - "123:4567:89ab:cdef:123:4567:89ab:cdef"); - // v4 compatible must be interpreted as IPv4 - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("::0:127.0.0.1")).ToString(), "127.0.0.1"); - // v4 mapped must be interpreted as IPv4 - BOOST_CHECK_EQUAL(BoostAsioToCNetAddr(boost::asio::ip::address::from_string("::ffff:127.0.0.1")).ToString(), "127.0.0.1"); -} - BOOST_AUTO_TEST_CASE(rpc_ban) { BOOST_CHECK_NO_THROW(CallRPC(string("clearbanned"))); From 505b30ff0152f6384164b0673e3c1b172c3cf0ad Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Mon, 19 Jan 2015 22:47:44 -0500 Subject: [PATCH 05/41] libevent: add depends --- depends/packages/libevent.mk | 26 ++++++++++++++++++++++++++ depends/packages/packages.mk | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 depends/packages/libevent.mk diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk new file mode 100644 index 000000000..3388a2843 --- /dev/null +++ b/depends/packages/libevent.mk @@ -0,0 +1,26 @@ +package=libevent +$(package)_version=2.0.22 +$(package)_download_path=https://github.com/libevent/libevent/releases/download/release-2.0.22-stable +$(package)_file_name=$(package)-$($(package)_version)-stable.tar.gz +$(package)_sha256_hash=71c2c49f0adadacfdbe6332a372c38cf9c8b7895bb73dabeaa53cdcc1d4e1fa3 + +define $(package)_set_vars + $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress + $(package)_config_opts_release=--disable-debug-mode + $(package)_config_opts_linux=--with-pic +endef + +define $(package)_config_cmds + $($(package)_autoconf) +endef + +define $(package)_build_cmds + $(MAKE) +endef + +define $(package)_stage_cmds + $(MAKE) DESTDIR=$($(package)_staging_dir) install +endef + +define $(package)_postprocess_cmds +endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 6b47eb650..9d025a6b2 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -1,6 +1,6 @@ rust_packages := rust librustzcash zcash_packages := libsnark libgmp libsodium -packages := boost openssl zeromq $(zcash_packages) googletest googlemock +packages := boost openssl libevent zeromq $(zcash_packages) googletest googlemock native_packages := native_ccache wallet_packages=bdb From df377ca82c9844659fbe46d4ea0c3c942e8cddcf Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 20 Jan 2015 06:04:59 +0100 Subject: [PATCH 06/41] build: build-system changes for libevent --- configure.ac | 14 ++++++++++++++ src/Makefile.am | 7 +++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 6dd1bf210..70ba71601 100644 --- a/configure.ac +++ b/configure.ac @@ -687,6 +687,12 @@ if test x$use_pkgconfig = xyes; then [ PKG_CHECK_MODULES([SSL], [libssl],, [AC_MSG_ERROR(openssl not found.)]) PKG_CHECK_MODULES([CRYPTO], [libcrypto],,[AC_MSG_ERROR(libcrypto not found.)]) + if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then + PKG_CHECK_MODULES([EVENT], [libevent],, [AC_MSG_ERROR(libevent not found.)]) + if test x$TARGET_OS != xwindows; then + PKG_CHECK_MODULES([EVENT_PTHREADS], [libevent_pthreads],, [AC_MSG_ERROR(libevent_pthreads not found.)]) + fi + fi if test "x$use_zmq" = "xyes"; then PKG_CHECK_MODULES([ZMQ],[libzmq >= 4], @@ -709,6 +715,14 @@ else AC_CHECK_HEADER([openssl/ssl.h],, AC_MSG_ERROR(libssl headers missing),) AC_CHECK_LIB([ssl], [main],SSL_LIBS=-lssl, AC_MSG_ERROR(libssl missing)) + if test x$build_bitcoin_utils$build_bitcoind$bitcoin_enable_qt$use_tests != xnononono; then + AC_CHECK_HEADER([event2/event.h],, AC_MSG_ERROR(libevent headers missing),) + AC_CHECK_LIB([event],[main],EVENT_LIBS=-levent,AC_MSG_ERROR(libevent missing)) + if test x$TARGET_OS != xwindows; then + AC_CHECK_LIB([event_pthreads],[main],EVENT_PTHREADS_LIBS=-levent_pthreads,AC_MSG_ERROR(libevent_pthreads missing)) + fi + fi + if test "x$use_zmq" = "xyes"; then AC_CHECK_HEADER([zmq.h], [AC_DEFINE([ENABLE_ZMQ],[1],[Define to 1 to enable ZMQ functions])], diff --git a/src/Makefile.am b/src/Makefile.am index 20455c971..c476208a1 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -192,7 +192,7 @@ obj/build.h: FORCE libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h # server: zcashd -libbitcoin_server_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) +libbitcoin_server_a_CPPFLAGS = $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_server_a_SOURCES = \ sendalert.cpp \ addrman.cpp \ @@ -390,13 +390,15 @@ zcashd_LDADD += \ $(SSL_LIBS) \ $(CRYPTO_LIBS) \ $(MINIUPNPC_LIBS) \ + $(EVENT_PTHREADS_LIBS) \ + $(EVENT_LIBS) \ $(LIBZCASH) \ $(LIBBITCOIN_CRYPTO) \ $(LIBZCASH_LIBS) # bitcoin-cli binary # zcash_cli_SOURCES = bitcoin-cli.cpp -zcash_cli_CPPFLAGS = $(BITCOIN_INCLUDES) +zcash_cli_CPPFLAGS = $(BITCOIN_INCLUDES) $(EVENT_CFLAGS) zcash_cli_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) if TARGET_WINDOWS @@ -410,6 +412,7 @@ zcash_cli_LDADD = \ $(BOOST_LIBS) \ $(SSL_LIBS) \ $(CRYPTO_LIBS) \ + $(EVENT_LIBS) \ $(LIBZCASH) \ $(LIBBITCOIN_CRYPTO) \ $(LIBZCASH_LIBS) From cc14ac45f49685786f2a54c5c930f79a0ee2c878 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 27 Aug 2015 20:27:12 +0200 Subject: [PATCH 07/41] tests: GET requests cannot have request body, use POST in rest.py Sending a request body with GET request is not valid in HTTP spec, and not compatible with evhttpd. --- qa/rpc-tests/rest.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/qa/rpc-tests/rest.py b/qa/rpc-tests/rest.py index 16489c09d..d315c66d3 100755 --- a/qa/rpc-tests/rest.py +++ b/qa/rpc-tests/rest.py @@ -32,10 +32,20 @@ def deser_uint256(f): r += t << (i * 32) return r -#allows simple http get calls with a request body -def http_get_call(host, port, path, requestdata = '', response_object = 0): +#allows simple http get calls +def http_get_call(host, port, path, response_object = 0): conn = httplib.HTTPConnection(host, port) - conn.request('GET', path, requestdata) + conn.request('GET', path) + + if response_object: + return conn.getresponse() + + return conn.getresponse().read() + +#allows simple http post calls with a request body +def http_post_call(host, port, path, requestdata = '', response_object = 0): + conn = httplib.HTTPConnection(host, port) + conn.request('POST', path, requestdata) if response_object: return conn.getresponse() @@ -137,7 +147,7 @@ class RESTTest (BitcoinTestFramework): binaryRequest += binascii.unhexlify(vintx); binaryRequest += pack("i", 0); - bin_response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', binaryRequest) + bin_response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', binaryRequest) output = StringIO.StringIO() output.write(bin_response) output.seek(0) @@ -175,14 +185,14 @@ class RESTTest (BitcoinTestFramework): #do some invalid requests json_request = '{"checkmempool' - response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True) + response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True) assert_equal(response.status, 500) #must be a 500 because we send a invalid json request json_request = '{"checkmempool' - response = http_get_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', json_request, True) + response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'bin', json_request, True) assert_equal(response.status, 500) #must be a 500 because we send a invalid bin request - response = http_get_call(url.hostname, url.port, '/rest/getutxos/checkmempool'+self.FORMAT_SEPARATOR+'bin', '', True) + response = http_post_call(url.hostname, url.port, '/rest/getutxos/checkmempool'+self.FORMAT_SEPARATOR+'bin', '', True) assert_equal(response.status, 500) #must be a 500 because we send a invalid bin request #test limits @@ -190,14 +200,14 @@ class RESTTest (BitcoinTestFramework): for x in range(0, 20): json_request += txid+'-'+str(n)+'/' json_request = json_request.rstrip("/") - response = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) + response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) assert_equal(response.status, 500) #must be a 500 because we exceeding the limits json_request = '/checkmempool/' for x in range(0, 15): json_request += txid+'-'+str(n)+'/' json_request = json_request.rstrip("/"); - response = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) + response = http_post_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json', '', True) assert_equal(response.status, 200) #must be a 500 because we exceeding the limits self.nodes[0].generate(1) #generate block to not affect upcoming tests @@ -217,27 +227,27 @@ class RESTTest (BitcoinTestFramework): # this is 32 9-bit indices # check binary format - response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True) + response = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"bin", True) assert_equal(response.status, 200) assert_greater_than(int(response.getheader('content-length')), 177) response_str = response.read() # compare with block header - response_header = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"bin", "", True) + response_header = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"bin", True) assert_equal(response_header.status, 200) assert_equal(int(response_header.getheader('content-length')), 177) response_header_str = response_header.read() assert_equal(response_str[0:177], response_header_str) # check block hex format - response_hex = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"hex", "", True) + response_hex = http_get_call(url.hostname, url.port, '/rest/block/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True) assert_equal(response_hex.status, 200) assert_greater_than(int(response_hex.getheader('content-length')), 354) response_hex_str = response_hex.read() assert_equal(response_str.encode("hex")[0:354], response_hex_str[0:354]) # compare with hex block header - response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", "", True) + response_header_hex = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"hex", True) assert_equal(response_header_hex.status, 200) assert_greater_than(int(response_header_hex.getheader('content-length')), 354) response_header_hex_str = response_header_hex.read() @@ -250,7 +260,7 @@ class RESTTest (BitcoinTestFramework): assert_equal(block_json_obj['hash'], bb_hash) # compare with json block header - response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"json", "", True) + response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/1/'+bb_hash+self.FORMAT_SEPARATOR+"json", True) assert_equal(response_header_json.status, 200) response_header_json_str = response_header_json.read() json_obj = json.loads(response_header_json_str, parse_float=decimal.Decimal) @@ -274,7 +284,7 @@ class RESTTest (BitcoinTestFramework): #see if we can get 5 headers in one response self.nodes[1].generate(5) self.sync_all() - response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/5/'+bb_hash+self.FORMAT_SEPARATOR+"json", "", True) + response_header_json = http_get_call(url.hostname, url.port, '/rest/headers/5/'+bb_hash+self.FORMAT_SEPARATOR+"json", True) assert_equal(response_header_json.status, 200) response_header_json_str = response_header_json.read() json_obj = json.loads(response_header_json_str) @@ -287,7 +297,7 @@ class RESTTest (BitcoinTestFramework): assert_equal(json_obj['txid'], tx_hash) # check hex format response - hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", "", True) + hex_string = http_get_call(url.hostname, url.port, '/rest/tx/'+tx_hash+self.FORMAT_SEPARATOR+"hex", True) assert_equal(hex_string.status, 200) assert_greater_than(int(response.getheader('content-length')), 10) From afd64f76ea5a3f89c4320e04ea8f3b4a86a2c69b Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 23 Jan 2015 07:53:17 +0100 Subject: [PATCH 08/41] evhttpd implementation - *Replace usage of boost::asio with [libevent2](http://libevent.org/)*. boost::asio is not part of C++11, so unlike other boost there is no forwards-compatibility reason to stick with it. Together with #4738 (convert json_spirit to UniValue), this rids Bitcoin Core of the worst offenders with regard to compile-time slowness. - *Replace spit-and-duct-tape http server with evhttp*. Front-end http handling is handled by libevent, a work queue (with configurable depth and parallelism) is used to handle application requests. - *Wrap HTTP request in C++ class*; this makes the application code mostly HTTP-server-neutral - *Refactor RPC to move all http-specific code to a separate file*. Theoreticaly this can allow building without HTTP server but with another RPC backend, e.g. Qt's debug console (currently not implemented) or future RPC mechanisms people may want to use. - *HTTP dispatch mechanism*; services (e.g., RPC, REST) register which URL paths they want to handle. By using a proven, high-performance asynchronous networking library (also used by Tor) and HTTP server, problems such as #5674, #5655, #344 should be avoided. What works? bitcoind, bitcoin-cli, bitcoin-qt. Unit tests and RPC/REST tests pass. The aim for now is everything but SSL support. Configuration options: - `-rpcthreads`: repurposed as "number of work handler threads". Still defaults to 4. - `-rpcworkqueue`: maximum depth of work queue. When this is reached, new requests will return a 500 Internal Error. - `-rpctimeout`: inactivity time, in seconds, after which to disconnect a client. - `-debug=http`: low-level http activity logging --- src/Makefile.am | 4 + src/bitcoin-cli.cpp | 125 +++++++--- src/bitcoind.cpp | 9 +- src/httprpc.cpp | 201 +++++++++++++++ src/httprpc.h | 37 +++ src/httpserver.cpp | 586 ++++++++++++++++++++++++++++++++++++++++++++ src/httpserver.h | 138 +++++++++++ src/init.cpp | 44 +++- src/init.h | 2 + src/rest.cpp | 245 +++++++++--------- src/rpcprotocol.cpp | 229 ----------------- src/rpcprotocol.h | 87 +------ src/rpcserver.cpp | 556 ++++------------------------------------- src/rpcserver.h | 73 +++--- 14 files changed, 1296 insertions(+), 1040 deletions(-) create mode 100644 src/httprpc.cpp create mode 100644 src/httprpc.h create mode 100644 src/httpserver.cpp create mode 100644 src/httpserver.h diff --git a/src/Makefile.am b/src/Makefile.am index c476208a1..75ec72af6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -119,6 +119,8 @@ BITCOIN_CORE_H = \ eccryptoverify.h \ ecwrapper.h \ hash.h \ + httprpc.h \ + httpserver.h \ init.h \ key.h \ keystore.h \ @@ -203,6 +205,8 @@ libbitcoin_server_a_SOURCES = \ bloom.cpp \ chain.cpp \ checkpoints.cpp \ + httprpc.cpp \ + httpserver.cpp \ init.cpp \ leveldbwrapper.cpp \ main.cpp \ diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 0e692622b..7c462809e 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -11,6 +11,12 @@ #include "utilstrencodings.h" #include +#include + +#include +#include +#include +#include #include @@ -32,9 +38,6 @@ std::string HelpMessageCli() strUsage += HelpMessageOpt("-rpcuser=", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=", _("Password for JSON-RPC connections")); - strUsage += HelpMessageGroup(_("SSL options: (see the Bitcoin Wiki for SSL setup instructions)")); - strUsage += HelpMessageOpt("-rpcssl", _("Use OpenSSL (https) for JSON-RPC connections")); - return strUsage; } @@ -94,32 +97,75 @@ static bool AppInitRPC(int argc, char* argv[]) fprintf(stderr, "Error: Invalid combination of -regtest and -testnet.\n"); return false; } + if (GetBoolArg("-rpcssl", false)) + { + fprintf(stderr, "Error: SSL mode for RPC (-rpcssl) is no longer supported.\n"); + return false; + } return true; } + +/** Reply structure for request_done to fill in */ +struct HTTPReply +{ + int status; + std::string body; +}; + +static void http_request_done(struct evhttp_request *req, void *ctx) +{ + HTTPReply *reply = static_cast(ctx); + + if (req == NULL) { + /* If req is NULL, it means an error occurred while connecting, but + * I'm not sure how to find out which one. We also don't really care. + */ + reply->status = 0; + return; + } + + reply->status = evhttp_request_get_response_code(req); + + struct evbuffer *buf = evhttp_request_get_input_buffer(req); + if (buf) + { + size_t size = evbuffer_get_length(buf); + const char *data = (const char*)evbuffer_pullup(buf, size); + if (data) + reply->body = std::string(data, size); + evbuffer_drain(buf, size); + } +} + UniValue CallRPC(const string& strMethod, const UniValue& params) { - // Connect to localhost - bool fUseSSL = GetBoolArg("-rpcssl", false); - boost::asio::io_service io_service; - boost::asio::ssl::context context(io_service, boost::asio::ssl::context::sslv23); - context.set_options(boost::asio::ssl::context::no_sslv2 | boost::asio::ssl::context::no_sslv3); - boost::asio::ssl::stream sslStream(io_service, context); - SSLIOStreamDevice d(sslStream, fUseSSL); - boost::iostreams::stream< SSLIOStreamDevice > stream(d); + std::string host = GetArg("-rpcconnect", "127.0.0.1"); + int port = GetArg("-rpcport", BaseParams().RPCPort()); - const bool fConnected = d.connect(GetArg("-rpcconnect", "127.0.0.1"), GetArg("-rpcport", itostr(BaseParams().RPCPort()))); - if (!fConnected) - throw CConnectionFailed("couldn't connect to server"); + // Create event base + struct event_base *base = event_base_new(); // TODO RAII + if (!base) + throw runtime_error("cannot create event_base"); - // Find credentials to use + // Synchronously look up hostname + struct evhttp_connection *evcon = evhttp_connection_base_new(base, NULL, host.c_str(), port); // TODO RAII + if (evcon == NULL) + throw runtime_error("create connection failed"); + evhttp_connection_set_timeout(evcon, GetArg("-rpctimeout", 30)); + + HTTPReply response; + struct evhttp_request *req = evhttp_request_new(http_request_done, (void*)&response); // TODO RAII + if (req == NULL) + throw runtime_error("create http request failed"); + + // Get credentials std::string strRPCUserColonPass; if (mapArgs["-rpcpassword"] == "") { // Try fall back to cookie-based authentication if no password is provided if (!GetAuthCookie(&strRPCUserColonPass)) { throw runtime_error(strprintf( - _("You must set rpcpassword= in the configuration file:\n%s\n" - "If the file does not exist, create it with owner-readable-only file permissions."), + _("Could not locate RPC credentials. No authentication cookie could be found, and no rpcpassword is set in the configuration file (%s)"), GetConfigFile().string().c_str())); } @@ -127,34 +173,41 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]; } - // HTTP basic authentication - map mapRequestHeaders; - mapRequestHeaders["Authorization"] = string("Basic ") + EncodeBase64(strRPCUserColonPass); + struct evkeyvalq *output_headers = evhttp_request_get_output_headers(req); + assert(output_headers); + evhttp_add_header(output_headers, "Host", host.c_str()); + evhttp_add_header(output_headers, "Connection", "close"); + evhttp_add_header(output_headers, "Authorization", (std::string("Basic ") + EncodeBase64(strRPCUserColonPass)).c_str()); - // Send request - string strRequest = JSONRPCRequest(strMethod, params, 1); - string strPost = HTTPPost(strRequest, mapRequestHeaders); - stream << strPost << std::flush; + // Attach request data + std::string strRequest = JSONRPCRequest(strMethod, params, 1); + struct evbuffer * output_buffer = evhttp_request_get_output_buffer(req); + assert(output_buffer); + evbuffer_add(output_buffer, strRequest.data(), strRequest.size()); - // Receive HTTP reply status - int nProto = 0; - int nStatus = ReadHTTPStatus(stream, nProto); + int r = evhttp_make_request(evcon, req, EVHTTP_REQ_POST, "/"); + if (r != 0) { + evhttp_connection_free(evcon); + event_base_free(base); + throw CConnectionFailed("send http request failed"); + } - // Receive HTTP reply message headers and body - map mapHeaders; - string strReply; - ReadHTTPMessage(stream, mapHeaders, strReply, nProto, std::numeric_limits::max()); + event_base_dispatch(base); + evhttp_connection_free(evcon); + event_base_free(base); - if (nStatus == HTTP_UNAUTHORIZED) + if (response.status == 0) + throw CConnectionFailed("couldn't connect to server"); + else if (response.status == HTTP_UNAUTHORIZED) throw runtime_error("incorrect rpcuser or rpcpassword (authorization failed)"); - else if (nStatus >= 400 && nStatus != HTTP_BAD_REQUEST && nStatus != HTTP_NOT_FOUND && nStatus != HTTP_INTERNAL_SERVER_ERROR) - throw runtime_error(strprintf("server returned HTTP error %d", nStatus)); - else if (strReply.empty()) + else if (response.status >= 400 && response.status != HTTP_BAD_REQUEST && response.status != HTTP_NOT_FOUND && response.status != HTTP_INTERNAL_SERVER_ERROR) + throw runtime_error(strprintf("server returned HTTP error %d", response.status)); + else if (response.body.empty()) throw runtime_error("no response from server"); // Parse reply UniValue valReply(UniValue::VSTR); - if (!valReply.read(strReply)) + if (!valReply.read(response.body)) throw runtime_error("couldn't parse reply from server"); const UniValue& reply = valReply.get_obj(); if (reply.empty()) diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 2063f4c85..bfd47b93b 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -10,11 +10,16 @@ #include "noui.h" #include "scheduler.h" #include "util.h" +#include "httpserver.h" +#include "httprpc.h" +#include "rpcserver.h" #include #include #include +#include + /* Introduction text for doxygen: */ /*! \mainpage Developer documentation @@ -44,7 +49,7 @@ void WaitForShutdown(boost::thread_group* threadGroup) } if (threadGroup) { - threadGroup->interrupt_all(); + Interrupt(*threadGroup); threadGroup->join_all(); } } @@ -171,7 +176,7 @@ bool AppInit(int argc, char* argv[]) if (!fRet) { - threadGroup.interrupt_all(); + Interrupt(threadGroup); // threadGroup.join_all(); was left out intentionally here, because we didn't re-test all of // the startup-failure cases to make sure they don't result in a hang due to some // thread-blocking-waiting-for-another-thread-during-startup case diff --git a/src/httprpc.cpp b/src/httprpc.cpp new file mode 100644 index 000000000..570beadc5 --- /dev/null +++ b/src/httprpc.cpp @@ -0,0 +1,201 @@ +#include "httprpc.h" + +#include "base58.h" +#include "chainparams.h" +#include "httpserver.h" +#include "rpcprotocol.h" +#include "rpcserver.h" +#include "random.h" +#include "sync.h" +#include "util.h" +#include "utilstrencodings.h" +#include "ui_interface.h" + +#include // boost::trim + +/** Simple one-shot callback timer to be used by the RPC mechanism to e.g. + * re-lock the wellet. + */ +class HTTPRPCTimer : public RPCTimerBase +{ +public: + HTTPRPCTimer(struct event_base* eventBase, boost::function& func, int64_t seconds) : ev(eventBase, false, new Handler(func)) + { + struct timeval tv = {seconds, 0}; + ev.trigger(&tv); + } +private: + HTTPEvent ev; + + class Handler : public HTTPClosure + { + public: + Handler(const boost::function& func) : func(func) + { + } + private: + boost::function func; + void operator()() { func(); } + }; +}; + +class HTTPRPCTimerInterface : public RPCTimerInterface +{ +public: + HTTPRPCTimerInterface(struct event_base* base) : base(base) + { + } + const char* Name() + { + return "HTTP"; + } + RPCTimerBase* NewTimer(boost::function& func, int64_t seconds) + { + return new HTTPRPCTimer(base, func, seconds); + } +private: + struct event_base* base; +}; + + +/* Pre-base64-encoded authentication token */ +static std::string strRPCUserColonPass; +/* Stored RPC timer interface (for unregistration) */ +static HTTPRPCTimerInterface* httpRPCTimerInterface = 0; + +static void JSONErrorReply(HTTPRequest* req, const UniValue& objError, const UniValue& id) +{ + // Send error reply from json-rpc error object + int nStatus = HTTP_INTERNAL_SERVER_ERROR; + int code = find_value(objError, "code").get_int(); + + if (code == RPC_INVALID_REQUEST) + nStatus = HTTP_BAD_REQUEST; + else if (code == RPC_METHOD_NOT_FOUND) + nStatus = HTTP_NOT_FOUND; + + std::string strReply = JSONRPCReply(NullUniValue, objError, id); + + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(nStatus, strReply); +} + +static bool RPCAuthorized(const std::string& strAuth) +{ + if (strRPCUserColonPass.empty()) // Belt-and-suspenders measure if InitRPCAuthentication was not called + return false; + if (strAuth.substr(0, 6) != "Basic ") + return false; + std::string strUserPass64 = strAuth.substr(6); + boost::trim(strUserPass64); + std::string strUserPass = DecodeBase64(strUserPass64); + return TimingResistantEqual(strUserPass, strRPCUserColonPass); +} + +static bool HTTPReq_JSONRPC(HTTPRequest* req, const std::string &) +{ + // JSONRPC handles only POST + if (req->GetRequestMethod() != HTTPRequest::POST) { + req->WriteReply(HTTP_BAD_METHOD, "JSONRPC server handles only POST requests"); + return false; + } + // Check authorization + std::pair authHeader = req->GetHeader("authorization"); + if (!authHeader.first) { + req->WriteReply(HTTP_UNAUTHORIZED); + return false; + } + + if (!RPCAuthorized(authHeader.second)) { + LogPrintf("ThreadRPCServer incorrect password attempt from %s\n", req->GetPeer().ToString()); + + /* Deter brute-forcing + If this results in a DoS the user really + shouldn't have their RPC port exposed. */ + MilliSleep(250); + + req->WriteReply(HTTP_UNAUTHORIZED); + return false; + } + + JSONRequest jreq; + try { + // Parse request + UniValue valRequest; + if (!valRequest.read(req->ReadBody())) + throw JSONRPCError(RPC_PARSE_ERROR, "Parse error"); + + std::string strReply; + // singleton request + if (valRequest.isObject()) { + jreq.parse(valRequest); + + UniValue result = tableRPC.execute(jreq.strMethod, jreq.params); + + // Send reply + strReply = JSONRPCReply(result, NullUniValue, jreq.id); + + // array of requests + } else if (valRequest.isArray()) + strReply = JSONRPCExecBatch(valRequest.get_array()); + else + throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error"); + + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strReply); + } catch (const UniValue& objError) { + JSONErrorReply(req, objError, jreq.id); + return false; + } catch (const std::exception& e) { + JSONErrorReply(req, JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); + return false; + } + return true; +} + +static bool InitRPCAuthentication() +{ + if (mapArgs["-rpcpassword"] == "") + { + LogPrintf("No rpcpassword set - using random cookie authentication\n"); + if (!GenerateAuthCookie(&strRPCUserColonPass)) { + uiInterface.ThreadSafeMessageBox( + _("Error: A fatal internal error occurred, see debug.log for details"), // Same message as AbortNode + "", CClientUIInterface::MSG_ERROR); + return false; + } + } else { + strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]; + } + return true; +} + +bool StartHTTPRPC() +{ + LogPrint("rpc", "Starting HTTP RPC server\n"); + if (!InitRPCAuthentication()) + return false; + + RegisterHTTPHandler("/", true, HTTPReq_JSONRPC); + + assert(EventBase()); + httpRPCTimerInterface = new HTTPRPCTimerInterface(EventBase()); + RPCRegisterTimerInterface(httpRPCTimerInterface); + return true; +} + +void InterruptHTTPRPC() +{ + LogPrint("rpc", "Interrupting HTTP RPC server\n"); +} + +void StopHTTPRPC() +{ + LogPrint("rpc", "Stopping HTTP RPC server\n"); + UnregisterHTTPHandler("/", true); + if (httpRPCTimerInterface) { + RPCUnregisterTimerInterface(httpRPCTimerInterface); + delete httpRPCTimerInterface; + httpRPCTimerInterface = 0; + } +} diff --git a/src/httprpc.h b/src/httprpc.h new file mode 100644 index 000000000..d35445718 --- /dev/null +++ b/src/httprpc.h @@ -0,0 +1,37 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_HTTPRPC_H +#define BITCOIN_HTTPRPC_H + +#include +#include + +class HTTPRequest; + +/** Start HTTP RPC subsystem. + * Precondition; HTTP and RPC has been started. + */ +bool StartHTTPRPC(); +/** Interrupt HTTP RPC subsystem. + */ +void InterruptHTTPRPC(); +/** Stop HTTP RPC subsystem. + * Precondition; HTTP and RPC has been stopped. + */ +void StopHTTPRPC(); + +/** Start HTTP REST subsystem. + * Precondition; HTTP and RPC has been started. + */ +bool StartREST(); +/** Interrupt RPC REST subsystem. + */ +void InterruptREST(); +/** Stop HTTP REST subsystem. + * Precondition; HTTP and RPC has been stopped. + */ +void StopREST(); + +#endif diff --git a/src/httpserver.cpp b/src/httpserver.cpp new file mode 100644 index 000000000..89366b2e4 --- /dev/null +++ b/src/httpserver.cpp @@ -0,0 +1,586 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "httpserver.h" + +#include "chainparamsbase.h" +#include "compat.h" +#include "util.h" +#include "netbase.h" +#include "rpcprotocol.h" // For HTTP status codes +#include "sync.h" +#include "ui_interface.h" + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef EVENT__HAVE_NETINET_IN_H +#include +#ifdef _XOPEN_SOURCE_EXTENDED +#include +#endif +#endif + +#include // for to_lower() +#include +#include + +/** HTTP request work item */ +class HTTPWorkItem : public HTTPClosure +{ +public: + HTTPWorkItem(HTTPRequest* req, const std::string &path, const HTTPRequestHandler& func): + req(req), path(path), func(func) + { + } + void operator()() + { + func(req.get(), path); + } + + boost::scoped_ptr req; + +private: + std::string path; + HTTPRequestHandler func; +}; + +/** Simple work queue for distributing work over multiple threads. + * Work items are simply callable objects. + */ +template +class WorkQueue +{ +private: + /** Mutex protects entire object */ + CWaitableCriticalSection cs; + CConditionVariable cond; + /* XXX in C++11 we can use std::unique_ptr here and avoid manual cleanup */ + std::deque queue; + bool running; + size_t maxDepth; + +public: + WorkQueue(size_t maxDepth) : running(true), + maxDepth(maxDepth) + { + } + /* Precondition: worker threads have all stopped */ + ~WorkQueue() + { + while (!queue.empty()) { + delete queue.front(); + queue.pop_front(); + } + } + /** Enqueue a work item */ + bool Enqueue(WorkItem* item) + { + boost::unique_lock lock(cs); + if (queue.size() >= maxDepth) { + return false; + } + queue.push_back(item); + cond.notify_one(); + return true; + } + /** Thread function */ + void Run() + { + while (running) { + WorkItem* i = 0; + { + boost::unique_lock lock(cs); + while (running && queue.empty()) + cond.wait(lock); + if (!running) + break; + i = queue.front(); + queue.pop_front(); + } + (*i)(); + delete i; + } + } + /** Interrupt and exit loops */ + void Interrupt() + { + boost::unique_lock lock(cs); + running = false; + cond.notify_all(); + } + + /** Return current depth of queue */ + size_t Depth() + { + boost::unique_lock lock(cs); + return queue.size(); + } +}; + +struct HTTPPathHandler +{ + HTTPPathHandler() {} + HTTPPathHandler(std::string prefix, bool exactMatch, HTTPRequestHandler handler): + prefix(prefix), exactMatch(exactMatch), handler(handler) + { + } + std::string prefix; + bool exactMatch; + HTTPRequestHandler handler; +}; + +/** HTTP module state */ + +//! libevent event loop +static struct event_base* eventBase = 0; +//! HTTP server +struct evhttp* eventHTTP = 0; +//! List of subnets to allow RPC connections from +static std::vector rpc_allow_subnets; +//! Work queue for handling longer requests off the event loop thread +static WorkQueue* workQueue = 0; +//! Handlers for (sub)paths +std::vector pathHandlers; + +/** Check if a network address is allowed to access the HTTP server */ +static bool ClientAllowed(const CNetAddr& netaddr) +{ + if (!netaddr.IsValid()) + return false; + BOOST_FOREACH (const CSubNet& subnet, rpc_allow_subnets) + if (subnet.Match(netaddr)) + return true; + return false; +} + +/** Initialize ACL list for HTTP server */ +static bool InitHTTPAllowList() +{ + rpc_allow_subnets.clear(); + rpc_allow_subnets.push_back(CSubNet("127.0.0.0/8")); // always allow IPv4 local subnet + rpc_allow_subnets.push_back(CSubNet("::1")); // always allow IPv6 localhost + if (mapMultiArgs.count("-rpcallowip")) { + const std::vector& vAllow = mapMultiArgs["-rpcallowip"]; + BOOST_FOREACH (std::string strAllow, vAllow) { + CSubNet subnet(strAllow); + if (!subnet.IsValid()) { + uiInterface.ThreadSafeMessageBox( + strprintf("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24).", strAllow), + "", CClientUIInterface::MSG_ERROR); + return false; + } + rpc_allow_subnets.push_back(subnet); + } + } + std::string strAllowed; + BOOST_FOREACH (const CSubNet& subnet, rpc_allow_subnets) + strAllowed += subnet.ToString() + " "; + LogPrint("http", "Allowing HTTP connections from: %s\n", strAllowed); + return true; +} + +/** HTTP request method as string - use for logging only */ +static std::string RequestMethodString(HTTPRequest::RequestMethod m) +{ + switch (m) { + case HTTPRequest::GET: + return "GET"; + break; + case HTTPRequest::POST: + return "POST"; + break; + case HTTPRequest::HEAD: + return "HEAD"; + break; + case HTTPRequest::PUT: + return "PUT"; + break; + default: + return "unknown"; + } +} + +/** HTTP request callback */ +static void http_request_cb(struct evhttp_request* req, void* arg) +{ + std::auto_ptr hreq(new HTTPRequest(req)); + + LogPrint("http", "Received a %s request for %s from %s\n", + RequestMethodString(hreq->GetRequestMethod()), hreq->GetURI(), hreq->GetPeer().ToString()); + + // Early address-based allow check + if (!ClientAllowed(hreq->GetPeer())) { + hreq->WriteReply(HTTP_FORBIDDEN); + return; + } + + // Early reject unknown HTTP methods + if (hreq->GetRequestMethod() == HTTPRequest::UNKNOWN) { + hreq->WriteReply(HTTP_BADMETHOD); + return; + } + + // Find registered handler for prefix + std::string strURI = hreq->GetURI(); + std::string path; + std::vector::const_iterator i = pathHandlers.begin(); + std::vector::const_iterator iend = pathHandlers.end(); + for (; i != iend; ++i) { + bool match = false; + if (i->exactMatch) + match = (strURI == i->prefix); + else + match = (strURI.substr(0, i->prefix.size()) == i->prefix); + if (match) { + path = strURI.substr(i->prefix.size()); + break; + } + } + + // Dispatch to worker thread + if (i != iend) { + std::auto_ptr item(new HTTPWorkItem(hreq.release(), path, i->handler)); + assert(workQueue); + if (workQueue->Enqueue(item.get())) + item.release(); /* if true, queue took ownership */ + else + item->req->WriteReply(HTTP_INTERNAL, "Work queue depth exceeded"); + } else { + hreq->WriteReply(HTTP_NOTFOUND); + } +} + +/** Event dispatcher thread */ +static void ThreadHTTP(struct event_base* base, struct evhttp* http) +{ + RenameThread("bitcoin-http"); + LogPrint("http", "Entering http event loop\n"); + event_base_dispatch(base); + // Event loop will be interrupted by InterruptHTTPServer() + LogPrint("http", "Exited http event loop\n"); +} + +/** Bind HTTP server to specified addresses */ +static bool HTTPBindAddresses(struct evhttp* http) +{ + int defaultPort = GetArg("-rpcport", BaseParams().RPCPort()); + int nBound = 0; + std::vector > endpoints; + + // Determine what addresses to bind to + if (!mapArgs.count("-rpcallowip")) { // Default to loopback if not allowing external IPs + endpoints.push_back(std::make_pair("::1", defaultPort)); + endpoints.push_back(std::make_pair("127.0.0.1", defaultPort)); + if (mapArgs.count("-rpcbind")) { + LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n"); + } + } else if (mapArgs.count("-rpcbind")) { // Specific bind address + const std::vector& vbind = mapMultiArgs["-rpcbind"]; + for (std::vector::const_iterator i = vbind.begin(); i != vbind.end(); ++i) { + int port = defaultPort; + std::string host; + SplitHostPort(*i, port, host); + endpoints.push_back(std::make_pair(host, port)); + } + } else { // No specific bind address specified, bind to any + endpoints.push_back(std::make_pair("::", defaultPort)); + endpoints.push_back(std::make_pair("0.0.0.0", defaultPort)); + } + + // Bind addresses + for (std::vector >::iterator i = endpoints.begin(); i != endpoints.end(); ++i) { + LogPrint("http", "Binding RPC on address %s port %i\n", i->first, i->second); + if (evhttp_bind_socket(http, i->first.empty() ? NULL : i->first.c_str(), i->second) == 0) { + nBound += 1; + } else { + LogPrintf("Binding RPC on address %s port %i failed.\n", i->first, i->second); + } + } + return nBound > 0; +} + +/** Simple wrapper to set thread name and run work queue */ +static void HTTPWorkQueueRun(WorkQueue* queue) +{ + RenameThread("bitcoin-httpworker"); + queue->Run(); +} + +bool StartHTTPServer(boost::thread_group& threadGroup) +{ + struct evhttp* http = 0; + struct event_base* base = 0; + + if (!InitHTTPAllowList()) + return false; + + if (GetBoolArg("-rpcssl", false)) { + uiInterface.ThreadSafeMessageBox( + "SSL mode for RPC (-rpcssl) is no longer supported.", + "", CClientUIInterface::MSG_ERROR); + return false; + } + +#ifdef WIN32 + evthread_use_windows_threads(); +#else + evthread_use_pthreads(); +#endif + + base = event_base_new(); // XXX RAII + if (!base) { + LogPrintf("Couldn't create an event_base: exiting\n"); + return false; + } + + /* Create a new evhttp object to handle requests. */ + http = evhttp_new(base); // XXX RAII + if (!http) { + LogPrintf("couldn't create evhttp. Exiting.\n"); + event_base_free(base); + return false; + } + + evhttp_set_timeout(http, GetArg("-rpctimeout", 30)); + evhttp_set_max_body_size(http, MAX_SIZE); + evhttp_set_gencb(http, http_request_cb, NULL); + + if (!HTTPBindAddresses(http)) { + LogPrintf("Unable to bind any endpoint for RPC server\n"); + evhttp_free(http); + event_base_free(base); + return false; + } + + LogPrint("http", "Starting HTTP server\n"); + int workQueueDepth = std::max((long)GetArg("-rpcworkqueue", 16), 1L); + int rpcThreads = std::max((long)GetArg("-rpcthreads", 4), 1L); + LogPrintf("HTTP: creating work queue of depth %d and %d worker threads\n", workQueueDepth, rpcThreads); + workQueue = new WorkQueue(workQueueDepth); + + threadGroup.create_thread(boost::bind(&ThreadHTTP, base, http)); + + for (int i = 0; i < rpcThreads; i++) + threadGroup.create_thread(boost::bind(&HTTPWorkQueueRun, workQueue)); + + eventBase = base; + eventHTTP = http; + return true; +} + +void InterruptHTTPServer() +{ + LogPrint("http", "Interrupting HTTP server\n"); + if (eventBase) + event_base_loopbreak(eventBase); + if (workQueue) + workQueue->Interrupt(); +} + +void StopHTTPServer() +{ + LogPrint("http", "Stopping HTTP server\n"); + delete workQueue; + if (eventHTTP) { + evhttp_free(eventHTTP); + eventHTTP = 0; + } + if (eventBase) { + event_base_free(eventBase); + eventBase = 0; + } +} + +struct event_base* EventBase() +{ + return eventBase; +} + +static void httpevent_callback_fn(evutil_socket_t, short, void* data) +{ + // Static handler simply passes through execution flow to _handle method + ((HTTPEvent*)data)->_handle(); +} + +void HTTPEvent::_handle() +{ + (*handler)(); + if (deleteWhenTriggered) + delete this; +} + +HTTPEvent::HTTPEvent(struct event_base* base, bool deleteWhenTriggered, HTTPClosure* handler) : deleteWhenTriggered(deleteWhenTriggered), handler(handler) +{ + ev = event_new(base, -1, 0, httpevent_callback_fn, this); + assert(ev); +} +HTTPEvent::~HTTPEvent() +{ + event_free(ev); +} +void HTTPEvent::trigger(struct timeval* tv) +{ + if (tv == NULL) + event_active(ev, 0, 0); // immediately trigger event in main thread + else + evtimer_add(ev, tv); // trigger after timeval passed +} +HTTPRequest::HTTPRequest(struct evhttp_request* req) : req(req), + replySent(false) +{ +} +HTTPRequest::~HTTPRequest() +{ + if (!replySent) { + // Keep track of whether reply was sent to avoid request leaks + LogPrintf("%s: Unhandled request\n", __func__); + WriteReply(HTTP_INTERNAL, "Unhandled request"); + } + // evhttpd cleans up the request, as long as a reply was sent. +} + +std::pair HTTPRequest::GetHeader(const std::string& hdr) +{ + const struct evkeyvalq* headers = evhttp_request_get_input_headers(req); + assert(headers); + const char* val = evhttp_find_header(headers, hdr.c_str()); + if (val) + return std::make_pair(true, val); + else + return std::make_pair(false, ""); +} + +std::string HTTPRequest::ReadBody() +{ + struct evbuffer* buf = evhttp_request_get_input_buffer(req); + if (!buf) + return ""; + size_t size = evbuffer_get_length(buf); + /** Trivial implementation: if this is ever a performance bottleneck, + * internal copying can be avoided in multi-segment buffers by using + * evbuffer_peek and an awkward loop. Though in that case, it'd be even + * better to not copy into an intermediate string but use a stream + * abstraction to consume the evbuffer on the fly in the parsing algorithm. + */ + const char* data = (const char*)evbuffer_pullup(buf, size); + if (!data) // returns NULL in case of empty buffer + return ""; + std::string rv(data, size); + evbuffer_drain(buf, size); + return rv; +} + +void HTTPRequest::WriteHeader(const std::string& hdr, const std::string& value) +{ + struct evkeyvalq* headers = evhttp_request_get_output_headers(req); + assert(headers); + evhttp_add_header(headers, hdr.c_str(), value.c_str()); +} + +/** Closure sent to main thread to request a reply to be sent to + * a HTTP request. + * Replies must be sent in the main loop in the main http thread, + * this cannot be done from worker threads. + */ +struct HTTPSendReplyHandler : HTTPClosure { +public: + HTTPSendReplyHandler(struct evhttp_request* req, int nStatus) : req(req), nStatus(nStatus) + { + } + void operator()() + { + evhttp_send_reply(req, nStatus, NULL, NULL); + } +private: + struct evhttp_request* req; + int nStatus; +}; + +void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) +{ + assert(!replySent && req); + // Send event to main http thread to send reply message + struct evbuffer* evb = evhttp_request_get_output_buffer(req); + assert(evb); + evbuffer_add(evb, strReply.data(), strReply.size()); + HTTPEvent* ev = new HTTPEvent(eventBase, true, + new HTTPSendReplyHandler(req, nStatus)); + ev->trigger(0); + replySent = true; + req = 0; // transferred back to main thread +} + +CService HTTPRequest::GetPeer() +{ + evhttp_connection* con = evhttp_request_get_connection(req); + CService peer; + if (con) { + // evhttp retains ownership over returned address string + const char* address = ""; + uint16_t port = 0; + evhttp_connection_get_peer(con, (char**)&address, &port); + peer = CService(address, port); + } + return peer; +} + +std::string HTTPRequest::GetURI() +{ + return evhttp_request_get_uri(req); +} + +HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod() +{ + switch (evhttp_request_get_command(req)) { + case EVHTTP_REQ_GET: + return GET; + break; + case EVHTTP_REQ_POST: + return POST; + break; + case EVHTTP_REQ_HEAD: + return HEAD; + break; + case EVHTTP_REQ_PUT: + return PUT; + break; + default: + return UNKNOWN; + break; + } +} + +void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler) +{ + LogPrint("http", "Registering HTTP handler for %s (exactmath %d)\n", prefix, exactMatch); + pathHandlers.push_back(HTTPPathHandler(prefix, exactMatch, handler)); +} + +void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch) +{ + std::vector::iterator i = pathHandlers.begin(); + std::vector::iterator iend = pathHandlers.end(); + for (; i != iend; ++i) + if (i->prefix == prefix && i->exactMatch == exactMatch) + break; + if (i != iend) + { + LogPrint("http", "Unregistering HTTP handler for %s (exactmath %d)\n", prefix, exactMatch); + pathHandlers.erase(i); + } +} + diff --git a/src/httpserver.h b/src/httpserver.h new file mode 100644 index 000000000..c6a780419 --- /dev/null +++ b/src/httpserver.h @@ -0,0 +1,138 @@ +// Copyright (c) 2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_HTTPSERVER_H +#define BITCOIN_HTTPSERVER_H + +#include +#include +#include +#include +#include + +struct evhttp_request; +struct event_base; +class CService; +class HTTPRequest; + +/** Start HTTP server */ +bool StartHTTPServer(boost::thread_group& threadGroup); +/** Interrupt HTTP server threads */ +void InterruptHTTPServer(); +/** Stop HTTP server */ +void StopHTTPServer(); + +/** Handler for requests to a certain HTTP path */ +typedef boost::function HTTPRequestHandler; +/** Register handler for prefix. + * If multiple handlers match a prefix, the first-registered one will + * be invoked. + */ +void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler); +/** Unregister handler for prefix */ +void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch); + +/** Return evhttp event base. This can be used by submodules to + * queue timers or custom events. + */ +struct event_base* EventBase(); + +/** In-flight HTTP request. + * Thin C++ wrapper around evhttp_request. + */ +class HTTPRequest +{ +private: + struct evhttp_request* req; + bool replySent; + +public: + HTTPRequest(struct evhttp_request* req); + ~HTTPRequest(); + + enum RequestMethod { + UNKNOWN, + GET, + POST, + HEAD, + PUT + }; + + /** Get requested URI. + */ + std::string GetURI(); + + /** Get CService (address:ip) for the origin of the http request. + */ + CService GetPeer(); + + /** Get request method. + */ + RequestMethod GetRequestMethod(); + + /** + * Get the request header specified by hdr, or an empty string. + * Return an pair (isPresent,string). + */ + std::pair GetHeader(const std::string& hdr); + + /** + * Read request body. + * + * @note As this consumes the underlying buffer, call this only once. + * Repeated calls will return an empty string. + */ + std::string ReadBody(); + + /** + * Write output header. + * + * @note call this before calling WriteErrorReply or Reply. + */ + void WriteHeader(const std::string& hdr, const std::string& value); + + /** + * Write HTTP reply. + * nStatus is the HTTP status code to send. + * strReply is the body of the reply. Keep it empty to send a standard message. + * + * @note Can be called only once. As this will give the request back to the + * main thread, do not call any other HTTPRequest methods after calling this. + */ + void WriteReply(int nStatus, const std::string& strReply = ""); +}; + +/** Event handler closure. + */ +class HTTPClosure +{ +public: + virtual void operator()() = 0; + virtual ~HTTPClosure() {} +}; + +/** Event class. This can be used either as an cross-thread trigger or as a timer. + */ +class HTTPEvent +{ +public: + /** Create a new event */ + HTTPEvent(struct event_base* base, bool deleteWhenTriggered, HTTPClosure* handler); + ~HTTPEvent(); + + /** Trigger the event. If tv is 0, trigger it immediately. Otherwise trigger it after + * the given time has elapsed. + */ + void trigger(struct timeval* tv); + + /** Internal function for handling, do not call directly */ + void _handle(); + +private: + bool deleteWhenTriggered; + struct event* ev; + boost::scoped_ptr handler; +}; + +#endif // BITCOIN_HTTPSERVER_H diff --git a/src/init.cpp b/src/init.cpp index 0a35b2828..a9557f240 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -17,6 +17,8 @@ #include "checkpoints.h" #include "compat/sanity.h" #include "consensus/validation.h" +#include "httpserver.h" +#include "httprpc.h" #include "key.h" #include "main.h" #include "metrics.h" @@ -151,6 +153,15 @@ public: static CCoinsViewDB *pcoinsdbview = NULL; static CCoinsViewErrorCatcher *pcoinscatcher = NULL; +void Interrupt(boost::thread_group& threadGroup) +{ + InterruptHTTPServer(); + InterruptHTTPRPC(); + InterruptRPC(); + InterruptREST(); + threadGroup.interrupt_all(); +} + void Shutdown() { LogPrintf("%s: In progress...\n", __func__); @@ -165,7 +176,11 @@ void Shutdown() /// module was initialized. RenameThread("zcash-shutoff"); mempool.AddTransactionsUpdated(1); - StopRPCThreads(); + + StopHTTPRPC(); + StopREST(); + StopRPC(); + StopHTTPServer(); #ifdef ENABLE_WALLET if (pwalletMain) pwalletMain->Flush(false); @@ -467,17 +482,10 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-rpcport=", strprintf(_("Listen for JSON-RPC connections on (default: %u or testnet: %u)"), 8232, 18232)); strUsage += HelpMessageOpt("-rpcallowip=", _("Allow JSON-RPC connections from specified source. Valid for are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times")); strUsage += HelpMessageOpt("-rpcthreads=", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), 4)); - strUsage += HelpMessageOpt("-rpckeepalive", strprintf(_("RPC support for HTTP persistent connections (default: %d)"), 1)); // Disabled until we can lock notes and also tune performance of libsnark which by default uses multiple threads //strUsage += HelpMessageOpt("-rpcasyncthreads=", strprintf(_("Set the number of threads to service Async RPC calls (default: %d)"), 1)); - strUsage += HelpMessageGroup(_("RPC SSL options: (see the Bitcoin Wiki for SSL setup instructions)")); - strUsage += HelpMessageOpt("-rpcssl", _("Use OpenSSL (https) for JSON-RPC connections")); - strUsage += HelpMessageOpt("-rpcsslcertificatechainfile=", strprintf(_("Server certificate file (default: %s)"), "server.cert")); - strUsage += HelpMessageOpt("-rpcsslprivatekeyfile=", strprintf(_("Server private key (default: %s)"), "server.pem")); - strUsage += HelpMessageOpt("-rpcsslciphers=", strprintf(_("Acceptable ciphers (default: %s)"), "TLSv1.2+HIGH:TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!3DES:@STRENGTH")); - if (mode == HMM_BITCOIND) { strUsage += HelpMessageGroup(_("Metrics Options (only if -daemon and -printtoconsole are not set):")); strUsage += HelpMessageOpt("-showmetrics", _("Show metrics on stdout (default: 1 if running in a console, 0 otherwise)")); @@ -661,6 +669,21 @@ static void ZC_LoadParams() pzcashParams->setProvingKeyPath(pk_path.string()); } +bool AppInitServers(boost::thread_group& threadGroup) +{ + RPCServer::OnStopped(&OnRPCStopped); + RPCServer::OnPreCommand(&OnRPCPreCommand); + if (!StartHTTPServer(threadGroup)) + return false; + if (!StartRPC()) + return false; + if (!StartHTTPRPC()) + return false; + if (GetBoolArg("-rest", false) && !StartREST()) + return false; + return true; +} + /** Initialize bitcoin. * @pre Parameters should be parsed and config file should be read. */ @@ -1070,9 +1093,8 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) if (fServer) { uiInterface.InitMessage.connect(SetRPCWarmupStatus); - RPCServer::OnStopped(&OnRPCStopped); - RPCServer::OnPreCommand(&OnRPCPreCommand); - StartRPCThreads(); + if (!AppInitServers(threadGroup)) + return InitError(_("Unable to start HTTP server. See debug log for details.")); } int64_t nStart; diff --git a/src/init.h b/src/init.h index b8d5ec723..710e43c69 100644 --- a/src/init.h +++ b/src/init.h @@ -23,6 +23,8 @@ extern ZCJoinSplit* pzcashParams; void StartShutdown(); bool ShutdownRequested(); +/** Interrupt threads */ +void Interrupt(boost::thread_group& threadGroup); void Shutdown(); bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler); diff --git a/src/rest.cpp b/src/rest.cpp index 92450dd11..b8cc5462b 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -6,6 +6,7 @@ #include "primitives/block.h" #include "primitives/transaction.h" #include "main.h" +#include "httpserver.h" #include "rpcserver.h" #include "streams.h" #include "sync.h" @@ -55,13 +56,6 @@ struct CCoin { } }; -class RestErr -{ -public: - enum HTTPStatusCode status; - string message; -}; - extern void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry); extern UniValue blockToJSON(const CBlock& block, const CBlockIndex* blockindex, bool txDetails = false); extern UniValue mempoolInfoToJSON(); @@ -69,15 +63,14 @@ extern UniValue mempoolToJSON(bool fVerbose = false); extern void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fIncludeHex); extern UniValue blockheaderToJSON(const CBlockIndex* blockindex); -static RestErr RESTERR(enum HTTPStatusCode status, string message) +static bool RESTERR(HTTPRequest* req, enum HTTPStatusCode status, string message) { - RestErr re; - re.status = status; - re.message = message; - return re; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(status, message + "\r\n"); + return false; } -static enum RetFormat ParseDataFormat(vector& params, const string strReq) +static enum RetFormat ParseDataFormat(vector& params, const string& strReq) { boost::split(params, strReq, boost::is_any_of(".")); if (params.size() > 1) { @@ -114,28 +107,35 @@ static bool ParseHashStr(const string& strReq, uint256& v) return true; } -static bool rest_headers(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool CheckWarmup(HTTPRequest* req) { + std::string statusmessage; + if (RPCIsInWarmup(&statusmessage)) + return RESTERR(req, HTTP_SERVICE_UNAVAILABLE, "Service temporarily unavailable: " + statusmessage); + return true; +} + +static bool rest_headers(HTTPRequest* req, + const std::string& strURIPart) +{ + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); vector path; boost::split(path, params[0], boost::is_any_of("/")); if (path.size() != 2) - throw RESTERR(HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers//.."); + return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers//.."); long count = strtol(path[0].c_str(), NULL, 10); if (count < 1 || count > 2000) - throw RESTERR(HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); + return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); string hashStr = path[1]; uint256 hash; if (!ParseHashStr(hashStr, hash)) - throw RESTERR(HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); std::vector headers; headers.reserve(count); @@ -159,28 +159,29 @@ static bool rest_headers(AcceptedConnection* conn, switch (rf) { case RF_BINARY: { string binaryHeader = ssHeader.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, binaryHeader.size(), "application/octet-stream") << binaryHeader << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryHeader); return true; } case RF_HEX: { string strHex = HexStr(ssHeader.begin(), ssHeader.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } - case RF_JSON: { UniValue jsonHeaders(UniValue::VARR); BOOST_FOREACH(const CBlockIndex *pindex, headers) { jsonHeaders.push_back(blockheaderToJSON(pindex)); } string strJSON = jsonHeaders.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } - default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: .bin, .hex)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex)"); } } @@ -188,34 +189,33 @@ static bool rest_headers(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_block(AcceptedConnection* conn, +static bool rest_block(HTTPRequest* req, const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun, bool showTxDetails) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); string hashStr = params[0]; uint256 hash; if (!ParseHashStr(hashStr, hash)) - throw RESTERR(HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); CBlock block; CBlockIndex* pblockindex = NULL; { LOCK(cs_main); if (mapBlockIndex.count(hash) == 0) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); pblockindex = mapBlockIndex[hash]; if (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); if (!ReadBlockFromDisk(block, pblockindex)) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION); @@ -224,25 +224,28 @@ static bool rest_block(AcceptedConnection* conn, switch (rf) { case RF_BINARY: { string binaryBlock = ssBlock.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, binaryBlock.size(), "application/octet-stream") << binaryBlock << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryBlock); return true; } case RF_HEX: { string strHex = HexStr(ssBlock.begin(), ssBlock.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } case RF_JSON: { UniValue objBlock = blockToJSON(block, pblockindex, showTxDetails); string strJSON = objBlock.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } @@ -250,30 +253,20 @@ static bool rest_block(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_block_extended(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_block_extended(HTTPRequest* req, const std::string& strURIPart) { - return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, true); + return rest_block(req, strURIPart, true); } -static bool rest_block_notxdetails(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_block_notxdetails(HTTPRequest* req, const std::string& strURIPart) { - return rest_block(conn, strURIPart, strRequest, mapHeaders, fRun, false); + return rest_block(req, strURIPart, false); } -static bool rest_chaininfo(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_chaininfo(HTTPRequest* req, const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); @@ -282,11 +275,12 @@ static bool rest_chaininfo(AcceptedConnection* conn, UniValue rpcParams(UniValue::VARR); UniValue chainInfoObject = getblockchaininfo(rpcParams, false); string strJSON = chainInfoObject.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } @@ -294,12 +288,10 @@ static bool rest_chaininfo(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_mempool_info(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_mempool_info(HTTPRequest* req, const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); @@ -308,11 +300,12 @@ static bool rest_mempool_info(AcceptedConnection* conn, UniValue mempoolInfoObject = mempoolInfoToJSON(); string strJSON = mempoolInfoObject.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } @@ -320,12 +313,10 @@ static bool rest_mempool_info(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_mempool_contents(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_mempool_contents(HTTPRequest* req, const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); @@ -334,11 +325,12 @@ static bool rest_mempool_contents(AcceptedConnection* conn, UniValue mempoolObject = mempoolToJSON(true); string strJSON = mempoolObject.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: json)"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); } } @@ -346,24 +338,22 @@ static bool rest_mempool_contents(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_tx(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_tx(HTTPRequest* req, const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; const RetFormat rf = ParseDataFormat(params, strURIPart); string hashStr = params[0]; uint256 hash; if (!ParseHashStr(hashStr, hash)) - throw RESTERR(HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hashStr); CTransaction tx; uint256 hashBlock = uint256(); if (!GetTransaction(hash, tx, hashBlock, true)) - throw RESTERR(HTTP_NOT_FOUND, hashStr + " not found"); + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << tx; @@ -371,13 +361,15 @@ static bool rest_tx(AcceptedConnection* conn, switch (rf) { case RF_BINARY: { string binaryTx = ssTx.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, binaryTx.size(), "application/octet-stream") << binaryTx << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, binaryTx); return true; } case RF_HEX: { string strHex = HexStr(ssTx.begin(), ssTx.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } @@ -385,12 +377,13 @@ static bool rest_tx(AcceptedConnection* conn, UniValue objTx(UniValue::VOBJ); TxToJSON(tx, hashBlock, objTx); string strJSON = objTx.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } @@ -398,12 +391,10 @@ static bool rest_tx(AcceptedConnection* conn, return true; // continue to process further HTTP reqs on this cxn } -static bool rest_getutxos(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun) +static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart) { + if (!CheckWarmup(req)) + return false; vector params; enum RetFormat rf = ParseDataFormat(params, strURIPart); @@ -415,8 +406,9 @@ static bool rest_getutxos(AcceptedConnection* conn, } // throw exception in case of a empty request - if (strRequest.length() == 0 && uriParts.size() == 0) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + std::string strRequestMutable = req->ReadBody(); + if (strRequestMutable.length() == 0 && uriParts.size() == 0) + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); bool fInputParsed = false; bool fCheckMemPool = false; @@ -440,7 +432,7 @@ static bool rest_getutxos(AcceptedConnection* conn, std::string strOutput = uriParts[i].substr(uriParts[i].find("-")+1); if (!ParseInt32(strOutput, &nOutput) || !IsHex(strTxid)) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Parse error"); txid.SetHex(strTxid); vOutPoints.push_back(COutPoint(txid, (uint32_t)nOutput)); @@ -449,15 +441,13 @@ static bool rest_getutxos(AcceptedConnection* conn, if (vOutPoints.size() > 0) fInputParsed = true; else - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); } - string strRequestMutable = strRequest; //convert const string to string for allowing hex to bin converting - switch (rf) { case RF_HEX: { // convert hex to bin, continue then with bin part - std::vector strRequestV = ParseHex(strRequest); + std::vector strRequestV = ParseHex(strRequestMutable); strRequestMutable.assign(strRequestV.begin(), strRequestV.end()); } @@ -467,7 +457,7 @@ static bool rest_getutxos(AcceptedConnection* conn, if (strRequestMutable.size() > 0) { if (fInputParsed) //don't allow sending input over URI and HTTP RAW DATA - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Combination of URI scheme inputs and raw post data is not allowed"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Combination of URI scheme inputs and raw post data is not allowed"); CDataStream oss(SER_NETWORK, PROTOCOL_VERSION); oss << strRequestMutable; @@ -476,24 +466,24 @@ static bool rest_getutxos(AcceptedConnection* conn, } } catch (const std::ios_base::failure& e) { // abort in case of unreadable binary data - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Parse error"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Parse error"); } break; } case RF_JSON: { if (!fInputParsed) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, "Error: empty request"); break; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } // limit max outpoints if (vOutPoints.size() > MAX_GETUTXOS_OUTPOINTS) - throw RESTERR(HTTP_INTERNAL_SERVER_ERROR, strprintf("Error: max outpoints exceeded (max: %d, tried: %d)", MAX_GETUTXOS_OUTPOINTS, vOutPoints.size())); + return RESTERR(req, HTTP_INTERNAL_SERVER_ERROR, strprintf("Error: max outpoints exceeded (max: %d, tried: %d)", MAX_GETUTXOS_OUTPOINTS, vOutPoints.size())); // check spentness and form a bitmap (as well as a JSON capable human-readble string representation) vector bitmap; @@ -543,7 +533,8 @@ static bool rest_getutxos(AcceptedConnection* conn, ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; string ssGetUTXOResponseString = ssGetUTXOResponse.str(); - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, ssGetUTXOResponseString.size(), "application/octet-stream") << ssGetUTXOResponseString << std::flush; + req->WriteHeader("Content-Type", "application/octet-stream"); + req->WriteReply(HTTP_OK, ssGetUTXOResponseString); return true; } @@ -552,7 +543,8 @@ static bool rest_getutxos(AcceptedConnection* conn, ssGetUTXOResponse << chainActive.Height() << chainActive.Tip()->GetBlockHash() << bitmap << outs; string strHex = HexStr(ssGetUTXOResponse.begin(), ssGetUTXOResponse.end()) + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strHex, fRun, false, "text/plain") << std::flush; + req->WriteHeader("Content-Type", "text/plain"); + req->WriteReply(HTTP_OK, strHex); return true; } @@ -582,11 +574,12 @@ static bool rest_getutxos(AcceptedConnection* conn, // return json string string strJSON = objGetUTXOResponse.write() + "\n"; - conn->stream() << HTTPReply(HTTP_OK, strJSON, fRun) << std::flush; + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, strJSON); return true; } default: { - throw RESTERR(HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")"); } } @@ -596,11 +589,7 @@ static bool rest_getutxos(AcceptedConnection* conn, static const struct { const char* prefix; - bool (*handler)(AcceptedConnection* conn, - const std::string& strURIPart, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun); + bool (*handler)(HTTPRequest* req, const std::string& strReq); } uri_prefixes[] = { {"/rest/tx/", rest_tx}, {"/rest/block/notxdetails/", rest_block_notxdetails}, @@ -612,29 +601,19 @@ static const struct { {"/rest/getutxos", rest_getutxos}, }; -bool HTTPReq_REST(AcceptedConnection* conn, - const std::string& strURI, - const string& strRequest, - const std::map& mapHeaders, - bool fRun) +bool StartREST() { - try { - std::string statusmessage; - if (RPCIsInWarmup(&statusmessage)) - throw RESTERR(HTTP_SERVICE_UNAVAILABLE, "Service temporarily unavailable: " + statusmessage); - - for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) { - unsigned int plen = strlen(uri_prefixes[i].prefix); - if (strURI.substr(0, plen) == uri_prefixes[i].prefix) { - string strURIPart = strURI.substr(plen); - return uri_prefixes[i].handler(conn, strURIPart, strRequest, mapHeaders, fRun); - } - } - } catch (const RestErr& re) { - conn->stream() << HTTPReply(re.status, re.message + "\r\n", false, false, "text/plain") << std::flush; - return false; - } - - conn->stream() << HTTPError(HTTP_NOT_FOUND, false) << std::flush; - return false; + for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) + RegisterHTTPHandler(uri_prefixes[i].prefix, false, uri_prefixes[i].handler); + return true; +} + +void InterruptREST() +{ +} + +void StopREST() +{ + for (unsigned int i = 0; i < ARRAYLEN(uri_prefixes); i++) + UnregisterHTTPHandler(uri_prefixes[i].prefix, false); } diff --git a/src/rpcprotocol.cpp b/src/rpcprotocol.cpp index ecc1d952a..ac26bbdaa 100644 --- a/src/rpcprotocol.cpp +++ b/src/rpcprotocol.cpp @@ -5,7 +5,6 @@ #include "rpcprotocol.h" -#include "clientversion.h" #include "random.h" #include "tinyformat.h" #include "util.h" @@ -16,236 +15,8 @@ #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - using namespace std; -//! Number of bytes to allocate and read at most at once in post data -const size_t POST_READ_SIZE = 256 * 1024; - -/** - * HTTP protocol - * - * This ain't Apache. We're just using HTTP header for the length field - * and to be compatible with other JSON-RPC implementations. - */ - -string HTTPPost(const string& strMsg, const map& mapRequestHeaders) -{ - ostringstream s; - s << "POST / HTTP/1.1\r\n" - << "User-Agent: zcash-json-rpc/" << FormatFullVersion() << "\r\n" - << "Host: 127.0.0.1\r\n" - << "Content-Type: application/json\r\n" - << "Content-Length: " << strMsg.size() << "\r\n" - << "Connection: close\r\n" - << "Accept: application/json\r\n"; - BOOST_FOREACH(const PAIRTYPE(string, string)& item, mapRequestHeaders) - s << item.first << ": " << item.second << "\r\n"; - s << "\r\n" << strMsg; - - return s.str(); -} - -static string rfc1123Time() -{ - return DateTimeStrFormat("%a, %d %b %Y %H:%M:%S +0000", GetTime()); -} - -static const char *httpStatusDescription(int nStatus) -{ - switch (nStatus) { - case HTTP_OK: return "OK"; - case HTTP_BAD_REQUEST: return "Bad Request"; - case HTTP_FORBIDDEN: return "Forbidden"; - case HTTP_NOT_FOUND: return "Not Found"; - case HTTP_INTERNAL_SERVER_ERROR: return "Internal Server Error"; - default: return ""; - } -} - -string HTTPError(int nStatus, bool keepalive, bool headersOnly) -{ - if (nStatus == HTTP_UNAUTHORIZED) - return strprintf("HTTP/1.0 401 Authorization Required\r\n" - "Date: %s\r\n" - "Server: zcash-json-rpc/%s\r\n" - "WWW-Authenticate: Basic realm=\"jsonrpc\"\r\n" - "Content-Type: text/html\r\n" - "Content-Length: 296\r\n" - "\r\n" - "\r\n" - "\r\n" - "\r\n" - "Error\r\n" - "\r\n" - "\r\n" - "

401 Unauthorized.

\r\n" - "\r\n", rfc1123Time(), FormatFullVersion()); - - return HTTPReply(nStatus, httpStatusDescription(nStatus), keepalive, - headersOnly, "text/plain"); -} - -string HTTPReplyHeader(int nStatus, bool keepalive, size_t contentLength, const char *contentType) -{ - return strprintf( - "HTTP/1.1 %d %s\r\n" - "Date: %s\r\n" - "Connection: %s\r\n" - "Content-Length: %u\r\n" - "Content-Type: %s\r\n" - "Server: zcash-json-rpc/%s\r\n" - "\r\n", - nStatus, - httpStatusDescription(nStatus), - rfc1123Time(), - keepalive ? "keep-alive" : "close", - contentLength, - contentType, - FormatFullVersion()); -} - -string HTTPReply(int nStatus, const string& strMsg, bool keepalive, - bool headersOnly, const char *contentType) -{ - if (headersOnly) - { - return HTTPReplyHeader(nStatus, keepalive, 0, contentType); - } else { - return HTTPReplyHeader(nStatus, keepalive, strMsg.size(), contentType) + strMsg; - } -} - -bool ReadHTTPRequestLine(std::basic_istream& stream, int &proto, - string& http_method, string& http_uri) -{ - string str; - getline(stream, str); - - // HTTP request line is space-delimited - vector vWords; - boost::split(vWords, str, boost::is_any_of(" ")); - if (vWords.size() < 2) - return false; - - // HTTP methods permitted: GET, POST - http_method = vWords[0]; - if (http_method != "GET" && http_method != "POST") - return false; - - // HTTP URI must be an absolute path, relative to current host - http_uri = vWords[1]; - if (http_uri.size() == 0 || http_uri[0] != '/') - return false; - - // parse proto, if present - string strProto = ""; - if (vWords.size() > 2) - strProto = vWords[2]; - - proto = 0; - const char *ver = strstr(strProto.c_str(), "HTTP/1."); - if (ver != NULL) - proto = atoi(ver+7); - - return true; -} - -int ReadHTTPStatus(std::basic_istream& stream, int &proto) -{ - string str; - getline(stream, str); - vector vWords; - boost::split(vWords, str, boost::is_any_of(" ")); - if (vWords.size() < 2) - return HTTP_INTERNAL_SERVER_ERROR; - proto = 0; - const char *ver = strstr(str.c_str(), "HTTP/1."); - if (ver != NULL) - proto = atoi(ver+7); - return atoi(vWords[1].c_str()); -} - -int ReadHTTPHeaders(std::basic_istream& stream, map& mapHeadersRet) -{ - int nLen = 0; - while (true) - { - string str; - std::getline(stream, str); - if (str.empty() || str == "\r") - break; - string::size_type nColon = str.find(":"); - if (nColon != string::npos) - { - string strHeader = str.substr(0, nColon); - boost::trim(strHeader); - boost::to_lower(strHeader); - string strValue = str.substr(nColon+1); - boost::trim(strValue); - mapHeadersRet[strHeader] = strValue; - if (strHeader == "content-length") - nLen = atoi(strValue.c_str()); - } - } - return nLen; -} - - -int ReadHTTPMessage(std::basic_istream& stream, map& mapHeadersRet, string& strMessageRet, - int nProto, size_t max_size) -{ - mapHeadersRet.clear(); - strMessageRet = ""; - - // Read header - int nLen = ReadHTTPHeaders(stream, mapHeadersRet); - if (nLen < 0 || (size_t)nLen > max_size) - return HTTP_INTERNAL_SERVER_ERROR; - - // Read message - if (nLen > 0) - { - vector vch; - size_t ptr = 0; - while (ptr < (size_t)nLen) - { - size_t bytes_to_read = std::min((size_t)nLen - ptr, POST_READ_SIZE); - vch.resize(ptr + bytes_to_read); - stream.read(&vch[ptr], bytes_to_read); - if (!stream) // Connection lost while reading - return HTTP_INTERNAL_SERVER_ERROR; - ptr += bytes_to_read; - } - strMessageRet = string(vch.begin(), vch.end()); - } - - string sConHdr = mapHeadersRet["connection"]; - - if ((sConHdr != "close") && (sConHdr != "keep-alive")) - { - if (nProto >= 1) - mapHeadersRet["connection"] = "keep-alive"; - else - mapHeadersRet["connection"] = "close"; - } - - return HTTP_OK; -} - /** * JSON-RPC protocol. Bitcoin speaks version 1.0 for maximum compatibility, * but uses JSON-RPC 1.1/2.0 standards for parts of the 1.0 standard that were diff --git a/src/rpcprotocol.h b/src/rpcprotocol.h index 6a658ae96..816ce9ec5 100644 --- a/src/rpcprotocol.h +++ b/src/rpcprotocol.h @@ -10,10 +10,6 @@ #include #include #include -#include -#include -#include -#include #include #include @@ -26,6 +22,7 @@ enum HTTPStatusCode HTTP_UNAUTHORIZED = 401, HTTP_FORBIDDEN = 403, HTTP_NOT_FOUND = 404, + HTTP_BAD_METHOD = 405, HTTP_INTERNAL_SERVER_ERROR = 500, HTTP_SERVICE_UNAVAILABLE = 503, }; @@ -79,88 +76,6 @@ enum RPCErrorCode RPC_WALLET_ALREADY_UNLOCKED = -17, //! Wallet is already unlocked }; -/** - * IOStream device that speaks SSL but can also speak non-SSL - */ -template -class SSLIOStreamDevice : public boost::iostreams::device { -public: - SSLIOStreamDevice(boost::asio::ssl::stream &streamIn, bool fUseSSLIn) : stream(streamIn) - { - fUseSSL = fUseSSLIn; - fNeedHandshake = fUseSSLIn; - } - - void handshake(boost::asio::ssl::stream_base::handshake_type role) - { - if (!fNeedHandshake) return; - fNeedHandshake = false; - stream.handshake(role); - } - std::streamsize read(char* s, std::streamsize n) - { - handshake(boost::asio::ssl::stream_base::server); // HTTPS servers read first - if (fUseSSL) return stream.read_some(boost::asio::buffer(s, n)); - return stream.next_layer().read_some(boost::asio::buffer(s, n)); - } - std::streamsize write(const char* s, std::streamsize n) - { - handshake(boost::asio::ssl::stream_base::client); // HTTPS clients write first - if (fUseSSL) return boost::asio::write(stream, boost::asio::buffer(s, n)); - return boost::asio::write(stream.next_layer(), boost::asio::buffer(s, n)); - } - bool connect(const std::string& server, const std::string& port) - { - using namespace boost::asio::ip; - tcp::resolver resolver(stream.get_io_service()); - tcp::resolver::iterator endpoint_iterator; -#if BOOST_VERSION >= 104300 - try { -#endif - // The default query (flags address_configured) tries IPv6 if - // non-localhost IPv6 configured, and IPv4 if non-localhost IPv4 - // configured. - tcp::resolver::query query(server.c_str(), port.c_str()); - endpoint_iterator = resolver.resolve(query); -#if BOOST_VERSION >= 104300 - } catch (const boost::system::system_error&) { - // If we at first don't succeed, try blanket lookup (IPv4+IPv6 independent of configured interfaces) - tcp::resolver::query query(server.c_str(), port.c_str(), resolver_query_base::flags()); - endpoint_iterator = resolver.resolve(query); - } -#endif - boost::system::error_code error = boost::asio::error::host_not_found; - tcp::resolver::iterator end; - while (error && endpoint_iterator != end) - { - stream.lowest_layer().close(); - stream.lowest_layer().connect(*endpoint_iterator++, error); - } - if (error) - return false; - return true; - } - -private: - bool fNeedHandshake; - bool fUseSSL; - boost::asio::ssl::stream& stream; -}; - -std::string HTTPPost(const std::string& strMsg, const std::map& mapRequestHeaders); -std::string HTTPError(int nStatus, bool keepalive, - bool headerOnly = false); -std::string HTTPReplyHeader(int nStatus, bool keepalive, size_t contentLength, - const char *contentType = "application/json"); -std::string HTTPReply(int nStatus, const std::string& strMsg, bool keepalive, - bool headerOnly = false, - const char *contentType = "application/json"); -bool ReadHTTPRequestLine(std::basic_istream& stream, int &proto, - std::string& http_method, std::string& http_uri); -int ReadHTTPStatus(std::basic_istream& stream, int &proto); -int ReadHTTPHeaders(std::basic_istream& stream, std::map& mapHeadersRet); -int ReadHTTPMessage(std::basic_istream& stream, std::map& mapHeadersRet, - std::string& strMessageRet, int nProto, size_t max_size); std::string JSONRPCRequest(const std::string& strMethod, const UniValue& params, const UniValue& id); UniValue JSONRPCReplyObj(const UniValue& result, const UniValue& error, const UniValue& id); std::string JSONRPCReply(const UniValue& result, const UniValue& error, const UniValue& id); diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 6ede4e227..7a7d3967f 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -12,16 +12,12 @@ #include "ui_interface.h" #include "util.h" #include "utilstrencodings.h" -#ifdef ENABLE_WALLET -#include "wallet/wallet.h" -#endif #include "asyncrpcqueue.h" #include -#include -#include -#include +#include + #include #include #include @@ -30,28 +26,20 @@ #include #include #include +#include // for to_upper() -#include - -using namespace boost::asio; using namespace RPCServer; using namespace std; -static std::string strRPCUserColonPass; - static bool fRPCRunning = false; static bool fRPCInWarmup = true; static std::string rpcWarmupStatus("RPC server started"); static CCriticalSection cs_rpcWarmup; - -//! These are created by StartRPCThreads, destroyed in StopRPCThreads -static boost::asio::io_service* rpc_io_service = NULL; -static map > deadlineTimers; -static ssl::context* rpc_ssl_context = NULL; -static boost::thread_group* rpc_worker_group = NULL; -static boost::asio::io_service::work *rpc_dummy_work = NULL; -static std::vector rpc_allow_subnets; //!< List of subnets to allow RPC connections from -static std::vector< boost::shared_ptr > rpc_acceptors; +/* Timer-creating functions */ +static std::vector timerInterfaces; +/* Map of name to timer. + * @note Can be changed to std::unique_ptr when C++11 */ +static std::map > deadlineTimers; static struct CRPCSignals { @@ -172,7 +160,6 @@ vector ParseHexO(const UniValue& o, string strKey) return ParseHexV(find_value(o, strKey), strKey); } - /** * Note: This interface may still be subject to change. */ @@ -264,8 +251,6 @@ UniValue stop(const UniValue& params, bool fHelp) return "Zcash server stopping"; } - - /** * Call Table */ @@ -427,7 +412,7 @@ CRPCTable::CRPCTable() } } -const CRPCCommand *CRPCTable::operator[](const std::string& name) const +const CRPCCommand *CRPCTable::operator[](const std::string &name) const { map::const_iterator it = mapCommands.find(name); if (it == mapCommands.end()) @@ -435,320 +420,9 @@ const CRPCCommand *CRPCTable::operator[](const std::string& name) const return (*it).second; } - -bool HTTPAuthorized(map& mapHeaders) +bool StartRPC() { - string strAuth = mapHeaders["authorization"]; - if (strAuth.substr(0,6) != "Basic ") - return false; - string strUserPass64 = strAuth.substr(6); boost::trim(strUserPass64); - string strUserPass = DecodeBase64(strUserPass64); - return TimingResistantEqual(strUserPass, strRPCUserColonPass); -} - -void ErrorReply(std::ostream& stream, const UniValue& objError, const UniValue& id) -{ - // Send error reply from json-rpc error object - int nStatus = HTTP_INTERNAL_SERVER_ERROR; - int code = find_value(objError, "code").get_int(); - if (code == RPC_INVALID_REQUEST) nStatus = HTTP_BAD_REQUEST; - else if (code == RPC_METHOD_NOT_FOUND) nStatus = HTTP_NOT_FOUND; - string strReply = JSONRPCReply(NullUniValue, objError, id); - stream << HTTPReply(nStatus, strReply, false) << std::flush; -} - -CNetAddr BoostAsioToCNetAddr(boost::asio::ip::address address) -{ - CNetAddr netaddr; - // Make sure that IPv4-compatible and IPv4-mapped IPv6 addresses are treated as IPv4 addresses - if (address.is_v6() - && (address.to_v6().is_v4_compatible() - || address.to_v6().is_v4_mapped())) - address = address.to_v6().to_v4(); - - if(address.is_v4()) - { - boost::asio::ip::address_v4::bytes_type bytes = address.to_v4().to_bytes(); - netaddr.SetRaw(NET_IPV4, &bytes[0]); - } - else - { - boost::asio::ip::address_v6::bytes_type bytes = address.to_v6().to_bytes(); - netaddr.SetRaw(NET_IPV6, &bytes[0]); - } - return netaddr; -} - -bool ClientAllowed(const boost::asio::ip::address& address) -{ - CNetAddr netaddr = BoostAsioToCNetAddr(address); - BOOST_FOREACH(const CSubNet &subnet, rpc_allow_subnets) - if (subnet.Match(netaddr)) - return true; - return false; -} - -template -class AcceptedConnectionImpl : public AcceptedConnection -{ -public: - AcceptedConnectionImpl( - boost::asio::io_service& io_service, - ssl::context &context, - bool fUseSSL) : - sslStream(io_service, context), - _d(sslStream, fUseSSL), - _stream(_d) - { - } - - virtual std::iostream& stream() - { - return _stream; - } - - virtual std::string peer_address_to_string() const - { - return peer.address().to_string(); - } - - virtual void close() - { - _stream.close(); - } - - typename Protocol::endpoint peer; - boost::asio::ssl::stream sslStream; - -private: - SSLIOStreamDevice _d; - boost::iostreams::stream< SSLIOStreamDevice > _stream; -}; - -void ServiceConnection(AcceptedConnection *conn); - -//! Forward declaration required for RPCListen -template -static void RPCAcceptHandler(boost::shared_ptr< basic_socket_acceptor > acceptor, - ssl::context& context, - bool fUseSSL, - boost::shared_ptr< AcceptedConnection > conn, - const boost::system::error_code& error); - -/** - * Sets up I/O resources to accept and handle a new connection. - */ -template -static void RPCListen(boost::shared_ptr< basic_socket_acceptor > acceptor, - ssl::context& context, - const bool fUseSSL) -{ - // Accept connection - boost::shared_ptr< AcceptedConnectionImpl > conn(new AcceptedConnectionImpl(acceptor->get_io_service(), context, fUseSSL)); - - acceptor->async_accept( - conn->sslStream.lowest_layer(), - conn->peer, - boost::bind(&RPCAcceptHandler, - acceptor, - boost::ref(context), - fUseSSL, - conn, - _1)); -} - - -/** - * Accept and handle incoming connection. - */ -template -static void RPCAcceptHandler(boost::shared_ptr< basic_socket_acceptor > acceptor, - ssl::context& context, - const bool fUseSSL, - boost::shared_ptr< AcceptedConnection > conn, - const boost::system::error_code& error) -{ - // Immediately start accepting new connections, except when we're cancelled or our socket is closed. - if (error != boost::asio::error::operation_aborted && acceptor->is_open()) - RPCListen(acceptor, context, fUseSSL); - - AcceptedConnectionImpl* tcp_conn = dynamic_cast< AcceptedConnectionImpl* >(conn.get()); - - if (error) - { - // TODO: Actually handle errors - LogPrintf("%s: Error: %s\n", __func__, error.message()); - } - // Restrict callers by IP. It is important to - // do this before starting client thread, to filter out - // certain DoS and misbehaving clients. - else if (tcp_conn && !ClientAllowed(tcp_conn->peer.address())) - { - // Only send a 403 if we're not using SSL to prevent a DoS during the SSL handshake. - if (!fUseSSL) - conn->stream() << HTTPError(HTTP_FORBIDDEN, false) << std::flush; - conn->close(); - } - else { - ServiceConnection(conn.get()); - conn->close(); - } -} - -static ip::tcp::endpoint ParseEndpoint(const std::string &strEndpoint, int defaultPort) -{ - std::string addr; - int port = defaultPort; - SplitHostPort(strEndpoint, port, addr); - return ip::tcp::endpoint(boost::asio::ip::address::from_string(addr), port); -} - -void StartRPCThreads() -{ - rpc_allow_subnets.clear(); - rpc_allow_subnets.push_back(CSubNet("127.0.0.0/8")); // always allow IPv4 local subnet - rpc_allow_subnets.push_back(CSubNet("::1")); // always allow IPv6 localhost - if (mapMultiArgs.count("-rpcallowip")) - { - const vector& vAllow = mapMultiArgs["-rpcallowip"]; - BOOST_FOREACH(string strAllow, vAllow) - { - CSubNet subnet(strAllow); - if(!subnet.IsValid()) - { - uiInterface.ThreadSafeMessageBox( - strprintf("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24).", strAllow), - "", CClientUIInterface::MSG_ERROR); - StartShutdown(); - return; - } - rpc_allow_subnets.push_back(subnet); - } - } - std::string strAllowed; - BOOST_FOREACH(const CSubNet &subnet, rpc_allow_subnets) - strAllowed += subnet.ToString() + " "; - LogPrint("rpc", "Allowing RPC connections from: %s\n", strAllowed); - - if (mapArgs["-rpcpassword"] == "") - { - LogPrintf("No rpcpassword set - using random cookie authentication\n"); - if (!GenerateAuthCookie(&strRPCUserColonPass)) { - uiInterface.ThreadSafeMessageBox( - _("Error: A fatal internal error occured, see debug.log for details"), // Same message as AbortNode - "", CClientUIInterface::MSG_ERROR); - StartShutdown(); - return; - } - } else { - strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]; - } - - assert(rpc_io_service == NULL); - rpc_io_service = new boost::asio::io_service(); - rpc_ssl_context = new ssl::context(*rpc_io_service, ssl::context::sslv23); - - const bool fUseSSL = GetBoolArg("-rpcssl", false); - - if (fUseSSL) - { - rpc_ssl_context->set_options(ssl::context::no_sslv2 | ssl::context::no_sslv3); - - boost::filesystem::path pathCertFile(GetArg("-rpcsslcertificatechainfile", "server.cert")); - if (!pathCertFile.is_complete()) pathCertFile = boost::filesystem::path(GetDataDir()) / pathCertFile; - if (boost::filesystem::exists(pathCertFile)) rpc_ssl_context->use_certificate_chain_file(pathCertFile.string()); - else LogPrintf("ThreadRPCServer ERROR: missing server certificate file %s\n", pathCertFile.string()); - - boost::filesystem::path pathPKFile(GetArg("-rpcsslprivatekeyfile", "server.pem")); - if (!pathPKFile.is_complete()) pathPKFile = boost::filesystem::path(GetDataDir()) / pathPKFile; - if (boost::filesystem::exists(pathPKFile)) rpc_ssl_context->use_private_key_file(pathPKFile.string(), ssl::context::pem); - else LogPrintf("ThreadRPCServer ERROR: missing server private key file %s\n", pathPKFile.string()); - - string strCiphers = GetArg("-rpcsslciphers", "TLSv1.2+HIGH:TLSv1+HIGH:!SSLv2:!aNULL:!eNULL:!3DES:@STRENGTH"); - SSL_CTX_set_cipher_list(rpc_ssl_context->impl(), strCiphers.c_str()); - } - - std::vector vEndpoints; - bool bBindAny = false; - int defaultPort = GetArg("-rpcport", BaseParams().RPCPort()); - if (!mapArgs.count("-rpcallowip")) // Default to loopback if not allowing external IPs - { - vEndpoints.push_back(ip::tcp::endpoint(boost::asio::ip::address_v6::loopback(), defaultPort)); - vEndpoints.push_back(ip::tcp::endpoint(boost::asio::ip::address_v4::loopback(), defaultPort)); - if (mapArgs.count("-rpcbind")) - { - LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n"); - } - } else if (mapArgs.count("-rpcbind")) // Specific bind address - { - BOOST_FOREACH(const std::string &addr, mapMultiArgs["-rpcbind"]) - { - try { - vEndpoints.push_back(ParseEndpoint(addr, defaultPort)); - } - catch (const boost::system::system_error&) - { - uiInterface.ThreadSafeMessageBox( - strprintf(_("Could not parse -rpcbind value %s as network address"), addr), - "", CClientUIInterface::MSG_ERROR); - StartShutdown(); - return; - } - } - } else { // No specific bind address specified, bind to any - vEndpoints.push_back(ip::tcp::endpoint(boost::asio::ip::address_v6::any(), defaultPort)); - vEndpoints.push_back(ip::tcp::endpoint(boost::asio::ip::address_v4::any(), defaultPort)); - // Prefer making the socket dual IPv6/IPv4 instead of binding - // to both addresses separately. - bBindAny = true; - } - - bool fListening = false; - std::string strerr; - std::string straddress; - BOOST_FOREACH(const ip::tcp::endpoint &endpoint, vEndpoints) - { - try { - boost::asio::ip::address bindAddress = endpoint.address(); - straddress = bindAddress.to_string(); - LogPrintf("Binding RPC on address %s port %i (IPv4+IPv6 bind any: %i)\n", straddress, endpoint.port(), bBindAny); - boost::system::error_code v6_only_error; - boost::shared_ptr acceptor(new ip::tcp::acceptor(*rpc_io_service)); - - acceptor->open(endpoint.protocol()); - acceptor->set_option(boost::asio::ip::tcp::acceptor::reuse_address(true)); - - // Try making the socket dual IPv6/IPv4 when listening on the IPv6 "any" address - acceptor->set_option(boost::asio::ip::v6_only( - !bBindAny || bindAddress != boost::asio::ip::address_v6::any()), v6_only_error); - - acceptor->bind(endpoint); - acceptor->listen(socket_base::max_connections); - - RPCListen(acceptor, *rpc_ssl_context, fUseSSL); - - fListening = true; - rpc_acceptors.push_back(acceptor); - // If dual IPv6/IPv4 bind successful, skip binding to IPv4 separately - if(bBindAny && bindAddress == boost::asio::ip::address_v6::any() && !v6_only_error) - break; - } - catch (const boost::system::system_error& e) - { - LogPrintf("ERROR: Binding RPC on address %s port %i failed: %s\n", straddress, endpoint.port(), e.what()); - strerr = strprintf(_("An error occurred while setting up the RPC address %s port %u for listening: %s"), straddress, endpoint.port(), e.what()); - } - } - - if (!fListening) { - uiInterface.ThreadSafeMessageBox(strerr, "", CClientUIInterface::MSG_ERROR); - StartShutdown(); - return; - } - - rpc_worker_group = new boost::thread_group(); - for (int i = 0; i < GetArg("-rpcthreads", 4); i++) - rpc_worker_group->create_thread(boost::bind(&boost::asio::io_service::run, rpc_io_service)); + LogPrint("rpc", "Starting RPC\n"); fRPCRunning = true; g_rpcSignals.Started(); @@ -766,57 +440,21 @@ void StartRPCThreads() for (int i = 0; i < n; i++) getAsyncRPCQueue()->addWorker(); */ + return true; } -void StartDummyRPCThread() +void InterruptRPC() { - if(rpc_io_service == NULL) - { - rpc_io_service = new boost::asio::io_service(); - /* Create dummy "work" to keep the thread from exiting when no timeouts active, - * see http://www.boost.org/doc/libs/1_51_0/doc/html/boost_asio/reference/io_service.html#boost_asio.reference.io_service.stopping_the_io_service_from_running_out_of_work */ - rpc_dummy_work = new boost::asio::io_service::work(*rpc_io_service); - rpc_worker_group = new boost::thread_group(); - rpc_worker_group->create_thread(boost::bind(&boost::asio::io_service::run, rpc_io_service)); - fRPCRunning = true; - } -} - -void StopRPCThreads() -{ - if (rpc_io_service == NULL) return; - // Set this to false first, so that longpolling loops will exit when woken up + LogPrint("rpc", "Interrupting RPC\n"); + // Interrupt e.g. running longpolls fRPCRunning = false; +} - // First, cancel all timers and acceptors - // This is not done automatically by ->stop(), and in some cases the destructor of - // boost::asio::io_service can hang if this is skipped. - boost::system::error_code ec; - BOOST_FOREACH(const boost::shared_ptr &acceptor, rpc_acceptors) - { - acceptor->cancel(ec); - if (ec) - LogPrintf("%s: Warning: %s when cancelling acceptor\n", __func__, ec.message()); - } - rpc_acceptors.clear(); - BOOST_FOREACH(const PAIRTYPE(std::string, boost::shared_ptr) &timer, deadlineTimers) - { - timer.second->cancel(ec); - if (ec) - LogPrintf("%s: Warning: %s when cancelling timer\n", __func__, ec.message()); - } +void StopRPC() +{ + LogPrint("rpc", "Stopping RPC\n"); deadlineTimers.clear(); - - DeleteAuthCookie(); - - rpc_io_service->stop(); g_rpcSignals.Stopped(); - if (rpc_worker_group != NULL) - rpc_worker_group->join_all(); - delete rpc_dummy_work; rpc_dummy_work = NULL; - delete rpc_worker_group; rpc_worker_group = NULL; - delete rpc_ssl_context; rpc_ssl_context = NULL; - delete rpc_io_service; rpc_io_service = NULL; // Tells async queue to cancel all operations and shutdown. LogPrintf("%s: waiting for async rpc workers to stop\n", __func__); @@ -849,36 +487,6 @@ bool RPCIsInWarmup(std::string *outStatus) return fRPCInWarmup; } -void RPCRunHandler(const boost::system::error_code& err, boost::function func) -{ - if (!err) - func(); -} - -void RPCRunLater(const std::string& name, boost::function func, int64_t nSeconds) -{ - assert(rpc_io_service != NULL); - - if (deadlineTimers.count(name) == 0) - { - deadlineTimers.insert(make_pair(name, - boost::shared_ptr(new deadline_timer(*rpc_io_service)))); - } - deadlineTimers[name]->expires_from_now(boost::posix_time::seconds(nSeconds)); - deadlineTimers[name]->async_wait(boost::bind(RPCRunHandler, _1, func)); -} - -class JSONRequest -{ -public: - UniValue id; - string strMethod; - UniValue params; - - JSONRequest() { id = NullUniValue; } - void parse(const UniValue& valRequest); -}; - void JSONRequest::parse(const UniValue& valRequest) { // Parse request @@ -909,7 +517,6 @@ void JSONRequest::parse(const UniValue& valRequest) throw JSONRPCError(RPC_INVALID_REQUEST, "Params must be an array"); } - static UniValue JSONRPCExecOne(const UniValue& req) { UniValue rpc_result(UniValue::VOBJ); @@ -934,7 +541,7 @@ static UniValue JSONRPCExecOne(const UniValue& req) return rpc_result; } -static string JSONRPCExecBatch(const UniValue& vReq) +std::string JSONRPCExecBatch(const UniValue& vReq) { UniValue ret(UniValue::VARR); for (size_t reqIdx = 0; reqIdx < vReq.size(); reqIdx++) @@ -943,107 +550,6 @@ static string JSONRPCExecBatch(const UniValue& vReq) return ret.write() + "\n"; } -static bool HTTPReq_JSONRPC(AcceptedConnection *conn, - string& strRequest, - map& mapHeaders, - bool fRun) -{ - // Check authorization - if (mapHeaders.count("authorization") == 0) - { - conn->stream() << HTTPError(HTTP_UNAUTHORIZED, false) << std::flush; - return false; - } - - if (!HTTPAuthorized(mapHeaders)) - { - LogPrintf("ThreadRPCServer incorrect password attempt from %s\n", conn->peer_address_to_string()); - /* Deter brute-forcing - We don't support exposing the RPC port, so this shouldn't result - in a DoS. */ - MilliSleep(250); - - conn->stream() << HTTPError(HTTP_UNAUTHORIZED, false) << std::flush; - return false; - } - - JSONRequest jreq; - try - { - // Parse request - UniValue valRequest; - if (!valRequest.read(strRequest)) - throw JSONRPCError(RPC_PARSE_ERROR, "Parse error"); - - string strReply; - - // singleton request - if (valRequest.isObject()) { - jreq.parse(valRequest); - - UniValue result = tableRPC.execute(jreq.strMethod, jreq.params); - - // Send reply - strReply = JSONRPCReply(result, NullUniValue, jreq.id); - - // array of requests - } else if (valRequest.isArray()) - strReply = JSONRPCExecBatch(valRequest.get_array()); - else - throw JSONRPCError(RPC_PARSE_ERROR, "Top-level object parse error"); - - conn->stream() << HTTPReplyHeader(HTTP_OK, fRun, strReply.size()) << strReply << std::flush; - } - catch (const UniValue& objError) - { - ErrorReply(conn->stream(), objError, jreq.id); - return false; - } - catch (const std::exception& e) - { - ErrorReply(conn->stream(), JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); - return false; - } - return true; -} - -void ServiceConnection(AcceptedConnection *conn) -{ - bool fRun = true; - while (fRun && !ShutdownRequested()) - { - int nProto = 0; - map mapHeaders; - string strRequest, strMethod, strURI; - - // Read HTTP request line - if (!ReadHTTPRequestLine(conn->stream(), nProto, strMethod, strURI)) - break; - - // Read HTTP message headers and body - ReadHTTPMessage(conn->stream(), mapHeaders, strRequest, nProto, MAX_SIZE); - - // HTTP Keep-Alive is false; close connection immediately - if ((mapHeaders["connection"] == "close") || (!GetBoolArg("-rpckeepalive", true))) - fRun = false; - - // Process via JSON-RPC API - if (strURI == "/") { - if (!HTTPReq_JSONRPC(conn, strRequest, mapHeaders, fRun)) - break; - - // Process via HTTP REST API - } else if (strURI.substr(0, 6) == "/rest/" && GetBoolArg("-rest", false)) { - if (!HTTPReq_REST(conn, strURI, strRequest, mapHeaders, fRun)) - break; - - } else { - conn->stream() << HTTPError(HTTP_NOT_FOUND, false) << std::flush; - break; - } - } -} - UniValue CRPCTable::execute(const std::string &strMethod, const UniValue ¶ms) const { // Return immediately if in warmup @@ -1084,6 +590,28 @@ std::string HelpExampleRpc(const std::string& methodname, const std::string& arg "\"method\": \"" + methodname + "\", \"params\": [" + args + "] }' -H 'content-type: text/plain;' http://127.0.0.1:8232/\n"; } +void RPCRegisterTimerInterface(RPCTimerInterface *iface) +{ + timerInterfaces.push_back(iface); +} + +void RPCUnregisterTimerInterface(RPCTimerInterface *iface) +{ + std::vector::iterator i = std::find(timerInterfaces.begin(), timerInterfaces.end(), iface); + assert(i != timerInterfaces.end()); + timerInterfaces.erase(i); +} + +void RPCRunLater(const std::string& name, boost::function func, int64_t nSeconds) +{ + if (timerInterfaces.empty()) + throw JSONRPCError(RPC_INTERNAL_ERROR, "No timer handler registered for RPC"); + deadlineTimers.erase(name); + RPCTimerInterface* timerInterface = timerInterfaces[0]; + LogPrint("rpc", "queue run of timer %s in %i seconds (using %s)\n", name, nSeconds, timerInterface->Name()); + deadlineTimers.insert(std::make_pair(name, timerInterface->NewTimer(func, nSeconds))); +} + const CRPCTable tableRPC; // Return async rpc queue diff --git a/src/rpcserver.h b/src/rpcserver.h index 911cbfff2..612075874 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -34,26 +34,17 @@ namespace RPCServer class CBlockIndex; class CNetAddr; -class AcceptedConnection +class JSONRequest { public: - virtual ~AcceptedConnection() {} + UniValue id; + std::string strMethod; + UniValue params; - virtual std::iostream& stream() = 0; - virtual std::string peer_address_to_string() const = 0; - virtual void close() = 0; + JSONRequest() { id = NullUniValue; } + void parse(const UniValue& valRequest); }; -/** Start RPC threads */ -void StartRPCThreads(); -/** - * Alternative to StartRPCThreads for the GUI, when no server is - * used. The RPC thread in this case is only used to handle timeouts. - * If real RPC threads have already been started this is a no-op. - */ -void StartDummyRPCThread(); -/** Stop RPC threads */ -void StopRPCThreads(); /** Query whether RPC is running */ bool IsRPCRunning(); @@ -87,15 +78,45 @@ void RPCTypeCheck(const UniValue& params, void RPCTypeCheckObj(const UniValue& o, const std::map& typesExpected, bool fAllowNull=false); +/** Opaque base class for timers returned by NewTimerFunc. + * This provides no methods at the moment, but makes sure that delete + * cleans up the whole state. + */ +class RPCTimerBase +{ +public: + virtual ~RPCTimerBase() {} +}; + /** - * Run func nSeconds from now. Uses boost deadline timers. + * RPC timer "driver". + */ +class RPCTimerInterface +{ +public: + virtual ~RPCTimerInterface() {} + /** Implementation name */ + virtual const char *Name() = 0; + /** Factory function for timers. + * RPC will call the function to create a timer that will call func in *seconds* seconds. + * @note As the RPC mechanism is backend-neutral, it can use different implementations of timers. + * This is needed to cope with the case in which there is no HTTP server, but + * only GUI RPC console, and to break the dependency of pcserver on httprpc. + */ + virtual RPCTimerBase* NewTimer(boost::function&, int64_t) = 0; +}; + +/** Register factory function for timers */ +void RPCRegisterTimerInterface(RPCTimerInterface *iface); +/** Unregister factory function for timers */ +void RPCUnregisterTimerInterface(RPCTimerInterface *iface); + +/** + * Run func nSeconds from now. * Overrides previous timer (if any). */ void RPCRunLater(const std::string& name, boost::function func, int64_t nSeconds); -//! Convert boost::asio address to CNetAddr -extern CNetAddr BoostAsioToCNetAddr(boost::asio::ip::address address); - typedef UniValue(*rpcfn_type)(const UniValue& params, bool fHelp); class CRPCCommand @@ -140,9 +161,6 @@ extern uint256 ParseHashO(const UniValue& o, std::string strKey); extern std::vector ParseHexV(const UniValue& v, std::string strName); extern std::vector ParseHexO(const UniValue& o, std::string strKey); -extern void InitRPCMining(); -extern void ShutdownRPCMining(); - extern int64_t nWalletUnlockTime; extern CAmount AmountFromValue(const UniValue& value); extern UniValue ValueFromAmount(const CAmount& amount); @@ -274,12 +292,9 @@ extern UniValue z_getoperationresult(const UniValue& params, bool fHelp); // in extern UniValue z_listoperationids(const UniValue& params, bool fHelp); // in rpcwallet.cpp extern UniValue z_validateaddress(const UniValue& params, bool fHelp); // in rpcmisc.cpp - -// in rest.cpp -extern bool HTTPReq_REST(AcceptedConnection *conn, - const std::string& strURI, - const std::string& strRequest, - const std::map& mapHeaders, - bool fRun); +bool StartRPC(); +void InterruptRPC(); +void StopRPC(); +std::string JSONRPCExecBatch(const UniValue& vReq); #endif // BITCOIN_RPCSERVER_H From 858afa1a4ac51e8e2760a998d8d6cb9f13563e96 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 28 Aug 2015 16:46:20 +0200 Subject: [PATCH 09/41] Implement RPCTimerHandler for Qt RPC console Implement RPCTimerHandler for Qt RPC console, so that `walletpassphrase` works with GUI and `-server=0`. Also simplify HTTPEvent-related code by using boost::function directly. Zcash: QT changes ommitted --- src/httprpc.cpp | 22 +++++++--------------- src/httpserver.cpp | 33 ++++++++------------------------- src/httpserver.h | 14 +++++++------- src/rpcserver.cpp | 2 +- src/rpcserver.h | 4 ++-- 5 files changed, 25 insertions(+), 50 deletions(-) diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 570beadc5..98ac750bb 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -19,24 +19,16 @@ class HTTPRPCTimer : public RPCTimerBase { public: - HTTPRPCTimer(struct event_base* eventBase, boost::function& func, int64_t seconds) : ev(eventBase, false, new Handler(func)) + HTTPRPCTimer(struct event_base* eventBase, boost::function& func, int64_t millis) : + ev(eventBase, false, func) { - struct timeval tv = {seconds, 0}; + struct timeval tv; + tv.tv_sec = millis/1000; + tv.tv_usec = (millis%1000)*1000; ev.trigger(&tv); } private: HTTPEvent ev; - - class Handler : public HTTPClosure - { - public: - Handler(const boost::function& func) : func(func) - { - } - private: - boost::function func; - void operator()() { func(); } - }; }; class HTTPRPCTimerInterface : public RPCTimerInterface @@ -49,9 +41,9 @@ public: { return "HTTP"; } - RPCTimerBase* NewTimer(boost::function& func, int64_t seconds) + RPCTimerBase* NewTimer(boost::function& func, int64_t millis) { - return new HTTPRPCTimer(base, func, seconds); + return new HTTPRPCTimer(base, func, millis); } private: struct event_base* base; diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 89366b2e4..13f870567 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -412,18 +412,15 @@ struct event_base* EventBase() static void httpevent_callback_fn(evutil_socket_t, short, void* data) { - // Static handler simply passes through execution flow to _handle method - ((HTTPEvent*)data)->_handle(); + // Static handler: simply call inner handler + HTTPEvent *self = ((HTTPEvent*)data); + self->handler(); + if (self->deleteWhenTriggered) + delete self; } -void HTTPEvent::_handle() -{ - (*handler)(); - if (deleteWhenTriggered) - delete this; -} - -HTTPEvent::HTTPEvent(struct event_base* base, bool deleteWhenTriggered, HTTPClosure* handler) : deleteWhenTriggered(deleteWhenTriggered), handler(handler) +HTTPEvent::HTTPEvent(struct event_base* base, bool deleteWhenTriggered, const boost::function& handler): + deleteWhenTriggered(deleteWhenTriggered), handler(handler) { ev = event_new(base, -1, 0, httpevent_callback_fn, this); assert(ev); @@ -496,20 +493,6 @@ void HTTPRequest::WriteHeader(const std::string& hdr, const std::string& value) * Replies must be sent in the main loop in the main http thread, * this cannot be done from worker threads. */ -struct HTTPSendReplyHandler : HTTPClosure { -public: - HTTPSendReplyHandler(struct evhttp_request* req, int nStatus) : req(req), nStatus(nStatus) - { - } - void operator()() - { - evhttp_send_reply(req, nStatus, NULL, NULL); - } -private: - struct evhttp_request* req; - int nStatus; -}; - void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) { assert(!replySent && req); @@ -518,7 +501,7 @@ void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) assert(evb); evbuffer_add(evb, strReply.data(), strReply.size()); HTTPEvent* ev = new HTTPEvent(eventBase, true, - new HTTPSendReplyHandler(req, nStatus)); + boost::bind(evhttp_send_reply, req, nStatus, (const char*)NULL, (struct evbuffer *)NULL)); ev->trigger(0); replySent = true; req = 0; // transferred back to main thread diff --git a/src/httpserver.h b/src/httpserver.h index c6a780419..648e8b6f8 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -117,8 +117,11 @@ public: class HTTPEvent { public: - /** Create a new event */ - HTTPEvent(struct event_base* base, bool deleteWhenTriggered, HTTPClosure* handler); + /** Create a new event. + * deleteWhenTriggered deletes this event object after the event is triggered (and the handler called) + * handler is the handler to call when the event is triggered. + */ + HTTPEvent(struct event_base* base, bool deleteWhenTriggered, const boost::function& handler); ~HTTPEvent(); /** Trigger the event. If tv is 0, trigger it immediately. Otherwise trigger it after @@ -126,13 +129,10 @@ public: */ void trigger(struct timeval* tv); - /** Internal function for handling, do not call directly */ - void _handle(); - -private: bool deleteWhenTriggered; + boost::function handler; +private: struct event* ev; - boost::scoped_ptr handler; }; #endif // BITCOIN_HTTPSERVER_H diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 7a7d3967f..ec1038dc5 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -609,7 +609,7 @@ void RPCRunLater(const std::string& name, boost::function func, int6 deadlineTimers.erase(name); RPCTimerInterface* timerInterface = timerInterfaces[0]; LogPrint("rpc", "queue run of timer %s in %i seconds (using %s)\n", name, nSeconds, timerInterface->Name()); - deadlineTimers.insert(std::make_pair(name, timerInterface->NewTimer(func, nSeconds))); + deadlineTimers.insert(std::make_pair(name, timerInterface->NewTimer(func, nSeconds*1000))); } const CRPCTable tableRPC; diff --git a/src/rpcserver.h b/src/rpcserver.h index 612075874..56fe2542e 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -98,12 +98,12 @@ public: /** Implementation name */ virtual const char *Name() = 0; /** Factory function for timers. - * RPC will call the function to create a timer that will call func in *seconds* seconds. + * RPC will call the function to create a timer that will call func in *millis* milliseconds. * @note As the RPC mechanism is backend-neutral, it can use different implementations of timers. * This is needed to cope with the case in which there is no HTTP server, but * only GUI RPC console, and to break the dependency of pcserver on httprpc. */ - virtual RPCTimerBase* NewTimer(boost::function&, int64_t) = 0; + virtual RPCTimerBase* NewTimer(boost::function& func, int64_t millis) = 0; }; /** Register factory function for timers */ From 9fb5b94e6416e3c19e94d265d74ee420be3b91af Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 28 Aug 2015 17:14:51 +0200 Subject: [PATCH 10/41] Document options for new HTTP/RPC server in --help --- src/httpserver.cpp | 6 +++--- src/httpserver.h | 4 ++++ src/init.cpp | 8 ++++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 13f870567..813764f22 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -355,7 +355,7 @@ bool StartHTTPServer(boost::thread_group& threadGroup) return false; } - evhttp_set_timeout(http, GetArg("-rpctimeout", 30)); + evhttp_set_timeout(http, GetArg("-rpctimeout", DEFAULT_HTTP_TIMEOUT)); evhttp_set_max_body_size(http, MAX_SIZE); evhttp_set_gencb(http, http_request_cb, NULL); @@ -367,8 +367,8 @@ bool StartHTTPServer(boost::thread_group& threadGroup) } LogPrint("http", "Starting HTTP server\n"); - int workQueueDepth = std::max((long)GetArg("-rpcworkqueue", 16), 1L); - int rpcThreads = std::max((long)GetArg("-rpcthreads", 4), 1L); + int workQueueDepth = std::max((long)GetArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); + int rpcThreads = std::max((long)GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); LogPrintf("HTTP: creating work queue of depth %d and %d worker threads\n", workQueueDepth, rpcThreads); workQueue = new WorkQueue(workQueueDepth); diff --git a/src/httpserver.h b/src/httpserver.h index 648e8b6f8..1b0d77ad4 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -11,6 +11,10 @@ #include #include +static const int DEFAULT_HTTP_THREADS=4; +static const int DEFAULT_HTTP_WORKQUEUE=16; +static const int DEFAULT_HTTP_TIMEOUT=30; + struct evhttp_request; struct event_base; class CService; diff --git a/src/init.cpp b/src/init.cpp index a9557f240..7560eeed7 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -420,7 +420,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", 1)); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", 0)); } - string debugCategories = "addrman, alert, bench, coindb, db, estimatefee, lock, mempool, net, partitioncheck, pow, proxy, prune, " + string debugCategories = "addrman, alert, bench, coindb, db, estimatefee, http, lock, mempool, net, partitioncheck, pow, proxy, prune, " "rand, reindex, rpc, selectcoins, zmq, zrpc, zrpcunsafe (implies zrpc)"; // Don't translate these strUsage += HelpMessageOpt("-debug=", strprintf(_("Output debugging information (default: %u, supplying is optional)"), 0) + ". " + _("If is not supplied or if = 1, output all debugging information.") + " " + _(" can be:") + " " + debugCategories + "."); @@ -481,7 +481,11 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-rpcpassword=", _("Password for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcport=", strprintf(_("Listen for JSON-RPC connections on (default: %u or testnet: %u)"), 8232, 18232)); strUsage += HelpMessageOpt("-rpcallowip=", _("Allow JSON-RPC connections from specified source. Valid for are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24). This option can be specified multiple times")); - strUsage += HelpMessageOpt("-rpcthreads=", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), 4)); + strUsage += HelpMessageOpt("-rpcthreads=", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), DEFAULT_HTTP_THREADS)); + if (showDebug) { + strUsage += HelpMessageOpt("-rpcworkqueue=", strprintf("Set the depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE)); + strUsage += HelpMessageOpt("-rpctimeout=", strprintf("Timeout during HTTP requests (default: %d)", DEFAULT_HTTP_TIMEOUT)); + } // Disabled until we can lock notes and also tune performance of libsnark which by default uses multiple threads //strUsage += HelpMessageOpt("-rpcasyncthreads=", strprintf(_("Set the number of threads to service Async RPC calls (default: %d)"), 1)); From 116503c0b8b29c0685aa8e86154c2aef79a6ee66 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 28 Aug 2015 16:55:16 +0200 Subject: [PATCH 11/41] Fix race condition between starting HTTP server thread and setting EventBase() Split StartHTTPServer into InitHTTPServer and StartHTTPServer to give clients a window to register their handlers without race conditions. Thanks @ajweiss for figuring this out. --- src/httpserver.cpp | 24 +++++++++++++++--------- src/httpserver.h | 9 ++++++++- src/init.cpp | 4 +++- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 813764f22..7e599b1d7 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -320,7 +320,7 @@ static void HTTPWorkQueueRun(WorkQueue* queue) queue->Run(); } -bool StartHTTPServer(boost::thread_group& threadGroup) +bool InitHTTPServer() { struct evhttp* http = 0; struct event_base* base = 0; @@ -366,19 +366,25 @@ bool StartHTTPServer(boost::thread_group& threadGroup) return false; } - LogPrint("http", "Starting HTTP server\n"); + LogPrint("http", "Initialized HTTP server\n"); int workQueueDepth = std::max((long)GetArg("-rpcworkqueue", DEFAULT_HTTP_WORKQUEUE), 1L); - int rpcThreads = std::max((long)GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); - LogPrintf("HTTP: creating work queue of depth %d and %d worker threads\n", workQueueDepth, rpcThreads); - workQueue = new WorkQueue(workQueueDepth); + LogPrintf("HTTP: creating work queue of depth %d\n", workQueueDepth); - threadGroup.create_thread(boost::bind(&ThreadHTTP, base, http)); + workQueue = new WorkQueue(workQueueDepth); + eventBase = base; + eventHTTP = http; + return true; +} + +bool StartHTTPServer(boost::thread_group& threadGroup) +{ + LogPrint("http", "Starting HTTP server\n"); + int rpcThreads = std::max((long)GetArg("-rpcthreads", DEFAULT_HTTP_THREADS), 1L); + LogPrintf("HTTP: starting %d worker threads\n", rpcThreads); + threadGroup.create_thread(boost::bind(&ThreadHTTP, eventBase, eventHTTP)); for (int i = 0; i < rpcThreads; i++) threadGroup.create_thread(boost::bind(&HTTPWorkQueueRun, workQueue)); - - eventBase = base; - eventHTTP = http; return true; } diff --git a/src/httpserver.h b/src/httpserver.h index 1b0d77ad4..459c60c04 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -20,7 +20,14 @@ struct event_base; class CService; class HTTPRequest; -/** Start HTTP server */ +/** Initialize HTTP server. + * Call this before RegisterHTTPHandler or EventBase(). + */ +bool InitHTTPServer(); +/** Start HTTP server. + * This is separate from InitHTTPServer to give users race-condition-free time + * to register their handlers between InitHTTPServer and StartHTTPServer. + */ bool StartHTTPServer(boost::thread_group& threadGroup); /** Interrupt HTTP server threads */ void InterruptHTTPServer(); diff --git a/src/init.cpp b/src/init.cpp index 7560eeed7..95cedd74c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -677,7 +677,7 @@ bool AppInitServers(boost::thread_group& threadGroup) { RPCServer::OnStopped(&OnRPCStopped); RPCServer::OnPreCommand(&OnRPCPreCommand); - if (!StartHTTPServer(threadGroup)) + if (!InitHTTPServer()) return false; if (!StartRPC()) return false; @@ -685,6 +685,8 @@ bool AppInitServers(boost::thread_group& threadGroup) return false; if (GetBoolArg("-rest", false) && !StartREST()) return false; + if (!StartHTTPServer(threadGroup)) + return false; return true; } From 91295c4b4dcb913a54bd233e1743c64efee12e6d Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Mon, 31 Aug 2015 11:17:26 +0200 Subject: [PATCH 12/41] libevent: Windows reuseaddr workaround in depends Make it possible to reuse sockets. This is necessary to make the RPC tests work in WINE. --- depends/packages/libevent.mk | 5 +++++ depends/patches/libevent/reuseaddr.patch | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 depends/patches/libevent/reuseaddr.patch diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 3388a2843..2e9be1e98 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -3,6 +3,11 @@ $(package)_version=2.0.22 $(package)_download_path=https://github.com/libevent/libevent/releases/download/release-2.0.22-stable $(package)_file_name=$(package)-$($(package)_version)-stable.tar.gz $(package)_sha256_hash=71c2c49f0adadacfdbe6332a372c38cf9c8b7895bb73dabeaa53cdcc1d4e1fa3 +$(package)_patches=reuseaddr.patch + +define $(package)_preprocess_cmds + patch -p1 < $($(package)_patch_dir)/reuseaddr.patch +endef define $(package)_set_vars $(package)_config_opts=--disable-shared --disable-openssl --disable-libevent-regress diff --git a/depends/patches/libevent/reuseaddr.patch b/depends/patches/libevent/reuseaddr.patch new file mode 100644 index 000000000..58695c11f --- /dev/null +++ b/depends/patches/libevent/reuseaddr.patch @@ -0,0 +1,21 @@ +--- old/evutil.c 2015-08-28 19:26:23.488765923 -0400 ++++ new/evutil.c 2015-08-28 19:27:41.392767019 -0400 +@@ -321,15 +321,16 @@ + int + evutil_make_listen_socket_reuseable(evutil_socket_t sock) + { +-#ifndef WIN32 + int one = 1; ++#ifndef WIN32 + /* REUSEADDR on Unix means, "don't hang on to this address after the + * listener is closed." On Windows, though, it means "don't keep other + * processes from binding to this address while we're using it. */ + return setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*) &one, + (ev_socklen_t)sizeof(one)); + #else +- return 0; ++ return setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*) &one, ++ (ev_socklen_t)sizeof(one)); + #endif + } + From 167b6231c93d40886de208cedccb9e7ed47f7502 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 2 Sep 2015 16:18:16 +0200 Subject: [PATCH 13/41] Move windows socket init to utility function --- src/bitcoin-cli.cpp | 4 ++++ src/init.cpp | 13 ++++--------- src/util.cpp | 12 ++++++++++++ src/util.h | 1 + 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 7c462809e..b227b9e05 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -303,6 +303,10 @@ int CommandLineRPC(int argc, char *argv[]) int main(int argc, char* argv[]) { SetupEnvironment(); + if (!SetupNetworking()) { + fprintf(stderr, "Error: Initializing networking failed\n"); + exit(1); + } try { if(!AppInitRPC(argc, argv)) diff --git a/src/init.cpp b/src/init.cpp index 95cedd74c..b7b7ffbd1 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -717,17 +717,12 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) typedef BOOL (WINAPI *PSETPROCDEPPOL)(DWORD); PSETPROCDEPPOL setProcDEPPol = (PSETPROCDEPPOL)GetProcAddress(GetModuleHandleA("Kernel32.dll"), "SetProcessDEPPolicy"); if (setProcDEPPol != NULL) setProcDEPPol(PROCESS_DEP_ENABLE); - - // Initialize Windows Sockets - WSADATA wsadata; - int ret = WSAStartup(MAKEWORD(2,2), &wsadata); - if (ret != NO_ERROR || LOBYTE(wsadata.wVersion ) != 2 || HIBYTE(wsadata.wVersion) != 2) - { - return InitError(strprintf("Error: Winsock library failed to start (WSAStartup returned error %d)", ret)); - } #endif -#ifndef WIN32 + if (!SetupNetworking()) + return InitError("Error: Initializing networking failed"); + +#ifndef WIN32 if (GetBoolArg("-sysperms", false)) { #ifdef ENABLE_WALLET if (!GetBoolArg("-disablewallet", false)) diff --git a/src/util.cpp b/src/util.cpp index 8ce727609..adfa6440e 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -863,6 +863,18 @@ void SetupEnvironment() boost::filesystem::path::imbue(loc); } +bool SetupNetworking() +{ +#ifdef WIN32 + // Initialize Windows Sockets + WSADATA wsadata; + int ret = WSAStartup(MAKEWORD(2,2), &wsadata); + if (ret != NO_ERROR || LOBYTE(wsadata.wVersion ) != 2 || HIBYTE(wsadata.wVersion) != 2) + return false; +#endif + return true; +} + void SetThreadPriority(int nPriority) { #ifdef WIN32 diff --git a/src/util.h b/src/util.h index e4eea2ba1..a0c16638c 100644 --- a/src/util.h +++ b/src/util.h @@ -64,6 +64,7 @@ inline std::string _(const char* psz) } void SetupEnvironment(); +bool SetupNetworking(); /** Return true if log accepts specified category */ bool LogAcceptCategory(const char* category); From 7fd5d4e30e3d01e301c083987886ffd86f2df7d7 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 4 Sep 2015 11:01:44 +0200 Subject: [PATCH 14/41] Revert "rpc-tests: re-enable rpc-tests for Windows" This reverts commit bd30c3dced21fca869a14c75081f15195762afe1. Disable windows RPC tests for now. These should be re-enabled once a suitable Wine version is used on Travis. --- qa/pull-tester/rpc-tests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index 08ff3fe7a..4650e1d52 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -8,6 +8,11 @@ CURDIR=$(cd $(dirname "$0"); pwd) export BITCOINCLI=${BUILDDIR}/qa/pull-tester/run-bitcoin-cli export BITCOIND=${REAL_BITCOIND} +if [ "x${EXEEXT}" = "x.exe" ]; then + echo "Win tests currently disabled" + exit 0 +fi + #Run the tests testScripts=( From 077ddc1187556e0ca459bb053a6659118c63306c Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Wed, 22 Feb 2017 16:46:55 +0000 Subject: [PATCH 15/41] Add libevent to zcash-gtest --- src/Makefile.gtest.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile.gtest.include b/src/Makefile.gtest.include index 8db5aea41..3752e0d48 100644 --- a/src/Makefile.gtest.include +++ b/src/Makefile.gtest.include @@ -51,7 +51,7 @@ if ENABLE_WALLET zcash_gtest_LDADD += $(LIBBITCOIN_WALLET) endif -zcash_gtest_LDADD += $(LIBZCASH_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(LIBZCASH) $(LIBZCASH_LIBS) +zcash_gtest_LDADD += $(LIBZCASH_CONSENSUS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(LIBZCASH) $(LIBZCASH_LIBS) zcash_gtest_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) -static From 7ed4d40c61205f50681a5ea1b9a62dc2b84ce561 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 15 Sep 2015 01:39:12 +0200 Subject: [PATCH 16/41] init: Ignore SIGPIPE Ignore SIGPIPE on all non-win32 OSes, otherwise an unexpectedly disconnecting RPC client will terminate the application. This problem was introduced with the libhttp-based RPC server. Fixes #6660. --- src/init.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index b7b7ffbd1..2fabac617 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -747,10 +747,8 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler) sa_hup.sa_flags = 0; sigaction(SIGHUP, &sa_hup, NULL); -#if defined (__SVR4) && defined (__sun) - // ignore SIGPIPE on Solaris + // Ignore SIGPIPE, otherwise it will bring the daemon down if the client closes unexpectedly signal(SIGPIPE, SIG_IGN); -#endif #endif // ********************************************************* Step 2: parameter interactions From a659991a2ca254280787739aa6df3b2382d030ec Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 18 Sep 2015 14:58:08 +0200 Subject: [PATCH 17/41] http: Disable libevent debug logging, if not explicitly enabled Add a option "-debug=libevent" to enable libevent debugging for troubleshooting. Libevent logging is redirected to our own log. --- src/httpserver.cpp | 19 +++++++++++++++++++ src/init.cpp | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 7e599b1d7..3ff175cd2 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -320,6 +320,15 @@ static void HTTPWorkQueueRun(WorkQueue* queue) queue->Run(); } +/** libevent event log callback */ +static void libevent_log_cb(int severity, const char *msg) +{ + if (severity >= EVENT_LOG_WARN) // Log warn messages and higher without debug category + LogPrintf("libevent: %s\n", msg); + else + LogPrint("libevent", "libevent: %s\n", msg); +} + bool InitHTTPServer() { struct evhttp* http = 0; @@ -335,6 +344,16 @@ bool InitHTTPServer() return false; } + // Redirect libevent's logging to our own log + event_set_log_callback(&libevent_log_cb); +#if LIBEVENT_VERSION_NUMBER >= 0x02010100 + // If -debug=libevent, set full libevent debugging. + // Otherwise, disable all libevent debugging. + if (LogAcceptCategory("libevent")) + event_enable_debug_logging(EVENT_DBG_ALL); + else + event_enable_debug_logging(EVENT_DBG_NONE); +#endif #ifdef WIN32 evthread_use_windows_threads(); #else diff --git a/src/init.cpp b/src/init.cpp index 2fabac617..4a3370879 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -420,7 +420,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", 1)); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", 0)); } - string debugCategories = "addrman, alert, bench, coindb, db, estimatefee, http, lock, mempool, net, partitioncheck, pow, proxy, prune, " + string debugCategories = "addrman, alert, bench, coindb, db, estimatefee, http, libevent, lock, mempool, net, partitioncheck, pow, proxy, prune, " "rand, reindex, rpc, selectcoins, zmq, zrpc, zrpcunsafe (implies zrpc)"; // Don't translate these strUsage += HelpMessageOpt("-debug=", strprintf(_("Output debugging information (default: %u, supplying is optional)"), 0) + ". " + _("If is not supplied or if = 1, output all debugging information.") + " " + _(" can be:") + " " + debugCategories + "."); From 89bccddcd8195059b64567bfb9c3cfe5ca0138cd Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 18 Sep 2015 15:45:38 +0200 Subject: [PATCH 18/41] rpc: Split option -rpctimeout into -rpcservertimeout and -rpcclienttimeout The two timeouts for the server and client, are essentially different: - In the case of the server it should be a lower value to avoid clients clogging up connection slots - In the case of the client it should be a high value to accomedate slow responses from the server, for example for slow queries or when the lock is contended Split the options into `-rpcservertimeout` and `-rpcclienttimeout` with respective defaults of 30 and 900. --- contrib/debian/examples/zcash.conf | 2 +- src/bitcoin-cli.cpp | 5 ++++- src/httpserver.cpp | 2 +- src/httpserver.h | 2 +- src/init.cpp | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/contrib/debian/examples/zcash.conf b/contrib/debian/examples/zcash.conf index cbe200496..0293500d0 100644 --- a/contrib/debian/examples/zcash.conf +++ b/contrib/debian/examples/zcash.conf @@ -73,7 +73,7 @@ # How many seconds zcash will wait for a complete RPC HTTP request. # after the HTTP connection is established. -#rpctimeout=30 +#rpcclienttimeout=30 # By default, only RPC connections from localhost are allowed. # Specify as many rpcallowip= settings as you like to allow connections from other hosts, diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index b227b9e05..7cad3f3b4 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -22,6 +22,8 @@ using namespace std; +static const int DEFAULT_HTTP_CLIENT_TIMEOUT=900; + std::string HelpMessageCli() { string strUsage; @@ -37,6 +39,7 @@ std::string HelpMessageCli() strUsage += HelpMessageOpt("-rpcwait", _("Wait for RPC server to start")); strUsage += HelpMessageOpt("-rpcuser=", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=", _("Password for JSON-RPC connections")); + strUsage += HelpMessageOpt("-rpcclienttimeout=", strprintf(_("Timeout during HTTP requests (default: %d)"), DEFAULT_HTTP_CLIENT_TIMEOUT)); return strUsage; } @@ -152,7 +155,7 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) struct evhttp_connection *evcon = evhttp_connection_base_new(base, NULL, host.c_str(), port); // TODO RAII if (evcon == NULL) throw runtime_error("create connection failed"); - evhttp_connection_set_timeout(evcon, GetArg("-rpctimeout", 30)); + evhttp_connection_set_timeout(evcon, GetArg("-rpcclienttimeout", DEFAULT_HTTP_CLIENT_TIMEOUT)); HTTPReply response; struct evhttp_request *req = evhttp_request_new(http_request_done, (void*)&response); // TODO RAII diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 3ff175cd2..19ab32390 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -374,7 +374,7 @@ bool InitHTTPServer() return false; } - evhttp_set_timeout(http, GetArg("-rpctimeout", DEFAULT_HTTP_TIMEOUT)); + evhttp_set_timeout(http, GetArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); evhttp_set_max_body_size(http, MAX_SIZE); evhttp_set_gencb(http, http_request_cb, NULL); diff --git a/src/httpserver.h b/src/httpserver.h index 459c60c04..b377dc19f 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -13,7 +13,7 @@ static const int DEFAULT_HTTP_THREADS=4; static const int DEFAULT_HTTP_WORKQUEUE=16; -static const int DEFAULT_HTTP_TIMEOUT=30; +static const int DEFAULT_HTTP_SERVER_TIMEOUT=30; struct evhttp_request; struct event_base; diff --git a/src/init.cpp b/src/init.cpp index 4a3370879..66f7005eb 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -484,7 +484,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-rpcthreads=", strprintf(_("Set the number of threads to service RPC calls (default: %d)"), DEFAULT_HTTP_THREADS)); if (showDebug) { strUsage += HelpMessageOpt("-rpcworkqueue=", strprintf("Set the depth of the work queue to service RPC calls (default: %d)", DEFAULT_HTTP_WORKQUEUE)); - strUsage += HelpMessageOpt("-rpctimeout=", strprintf("Timeout during HTTP requests (default: %d)", DEFAULT_HTTP_TIMEOUT)); + strUsage += HelpMessageOpt("-rpcservertimeout=", strprintf("Timeout during HTTP requests (default: %d)", DEFAULT_HTTP_SERVER_TIMEOUT)); } // Disabled until we can lock notes and also tune performance of libsnark which by default uses multiple threads From 7b41e725d08837d5524ec6feba31258425b55de8 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Sat, 19 Sep 2015 14:12:44 +0200 Subject: [PATCH 19/41] Make RPC tests cope with server-side timeout between requests Python's httplib does not graciously handle disconnections from the http server, resulting in BadStatusLine errors. See https://bugs.python.org/issue3566 "httplib persistent connections violate MUST in RFC2616 sec 8.1.4." This was fixed in Python 3.5. Work around it for now. --- qa/rpc-tests/test_framework/authproxy.py | 36 +++++++++++++++--------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index 1188f099e..111e6d76c 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -106,6 +106,26 @@ class AuthServiceProxy(object): name = "%s.%s" % (self.__service_name, name) return AuthServiceProxy(self.__service_url, name, connection=self.__conn) + def _request(self, method, path, postdata): + ''' + Do a HTTP request, with retry if we get disconnected (e.g. due to a timeout). + This is a workaround for https://bugs.python.org/issue3566 which is fixed in Python 3.5. + ''' + headers = {'Host': self.__url.hostname, + 'User-Agent': USER_AGENT, + 'Authorization': self.__auth_header, + 'Content-type': 'application/json'} + try: + self.__conn.request(method, path, postdata, headers) + return self._get_response() + except httplib.BadStatusLine as e: + if e.line == "''": # if connection was closed, try again + self.__conn.close() + self.__conn.request(method, path, postdata, headers) + return self._get_response() + else: + raise + def __call__(self, *args): AuthServiceProxy.__id_count += 1 @@ -115,13 +135,7 @@ class AuthServiceProxy(object): 'method': self.__service_name, 'params': args, 'id': AuthServiceProxy.__id_count}, default=EncodeDecimal) - self.__conn.request('POST', self.__url.path, postdata, - {'Host': self.__url.hostname, - 'User-Agent': USER_AGENT, - 'Authorization': self.__auth_header, - 'Content-type': 'application/json'}) - - response = self._get_response() + response = self._request('POST', self.__url.path, postdata) if response['error'] is not None: raise JSONRPCException(response['error']) elif 'result' not in response: @@ -133,13 +147,7 @@ class AuthServiceProxy(object): def _batch(self, rpc_call_list): postdata = json.dumps(list(rpc_call_list), default=EncodeDecimal) log.debug("--> "+postdata) - self.__conn.request('POST', self.__url.path, postdata, - {'Host': self.__url.hostname, - 'User-Agent': USER_AGENT, - 'Authorization': self.__auth_header, - 'Content-type': 'application/json'}) - - return self._get_response() + return self._request('POST', self.__url.path, postdata) def _get_response(self): http_response = self.__conn.getresponse() From c922edd00f376ab148be8e730f0d8768bcacbfb8 Mon Sep 17 00:00:00 2001 From: Bob McElrath Date: Wed, 28 Oct 2015 22:25:32 -0400 Subject: [PATCH 20/41] Add explicit shared_ptr constructor due to C++11 error --- src/rpcserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index ec1038dc5..2e8aa5aad 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -609,7 +609,7 @@ void RPCRunLater(const std::string& name, boost::function func, int6 deadlineTimers.erase(name); RPCTimerInterface* timerInterface = timerInterfaces[0]; LogPrint("rpc", "queue run of timer %s in %i seconds (using %s)\n", name, nSeconds, timerInterface->Name()); - deadlineTimers.insert(std::make_pair(name, timerInterface->NewTimer(func, nSeconds*1000))); + deadlineTimers.insert(std::make_pair(name, boost::shared_ptr(timerInterface->NewTimer(func, nSeconds*1000)))); } const CRPCTable tableRPC; From 599d2c4034eb36151e06c2adbd04318737a01e21 Mon Sep 17 00:00:00 2001 From: Gregory Maxwell Date: Sat, 14 Nov 2015 13:54:21 +0000 Subject: [PATCH 21/41] Avoid a compile error on hosts with libevent too old for EVENT_LOG_WARN. This uses _EVENT_LOG_WARN instead, which appears to be defined in the old versions of libevent that I have on some systems. --- src/httpserver.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 19ab32390..4eb9dcb54 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -323,6 +323,10 @@ static void HTTPWorkQueueRun(WorkQueue* queue) /** libevent event log callback */ static void libevent_log_cb(int severity, const char *msg) { +#ifndef EVENT_LOG_WARN +// EVENT_LOG_WARN was added in 2.0.19; but before then _EVENT_LOG_WARN existed. +# define EVENT_LOG_WARN _EVENT_LOG_WARN +#endif if (severity >= EVENT_LOG_WARN) // Log warn messages and higher without debug category LogPrintf("libevent: %s\n", msg); else From 0e2b1ae259f6f4ec3e4857469912b28f2e3c4513 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 28 Apr 2016 13:35:16 +0200 Subject: [PATCH 22/41] chain: define enum used as bit field as uint32_t Bitwise logic combined with `<` with undefined signedness will potentially results in undefined behavior. Fix this by defining the type as a c++11 typed enum. Fixes #6017. --- src/chain.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/chain.h b/src/chain.h index e4076e00a..b7e8a9176 100644 --- a/src/chain.h +++ b/src/chain.h @@ -56,7 +56,7 @@ struct CDiskBlockPos }; -enum BlockStatus { +enum BlockStatus: uint32_t { //! Unused. BLOCK_VALID_UNKNOWN = 0, From 08c581940ab2ee11734372ac98c9e65bb4a57c97 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 28 Apr 2016 13:40:20 +0200 Subject: [PATCH 23/41] =?UTF-8?q?auto=5Fptr=20=E2=86=92=20unique=5Fptr?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change the few occurrences of the deprecated `auto_ptr` to c++11 `unique_ptr`. Silences the deprecation warnings. Also add a missing `std::` for consistency. --- src/httpserver.cpp | 4 ++-- src/miner.cpp | 2 +- src/rpcmining.cpp | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 4eb9dcb54..c04c9da56 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -217,7 +217,7 @@ static std::string RequestMethodString(HTTPRequest::RequestMethod m) /** HTTP request callback */ static void http_request_cb(struct evhttp_request* req, void* arg) { - std::auto_ptr hreq(new HTTPRequest(req)); + std::unique_ptr hreq(new HTTPRequest(req)); LogPrint("http", "Received a %s request for %s from %s\n", RequestMethodString(hreq->GetRequestMethod()), hreq->GetURI(), hreq->GetPeer().ToString()); @@ -253,7 +253,7 @@ static void http_request_cb(struct evhttp_request* req, void* arg) // Dispatch to worker thread if (i != iend) { - std::auto_ptr item(new HTTPWorkItem(hreq.release(), path, i->handler)); + std::unique_ptr item(new HTTPWorkItem(hreq.release(), path, i->handler)); assert(workQueue); if (workQueue->Enqueue(item.get())) item.release(); /* if true, queue took ownership */ diff --git a/src/miner.cpp b/src/miner.cpp index 5e2811504..5e03ed9c0 100644 --- a/src/miner.cpp +++ b/src/miner.cpp @@ -110,7 +110,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn) { const CChainParams& chainparams = Params(); // Create new block - unique_ptr pblocktemplate(new CBlockTemplate()); + std::unique_ptr pblocktemplate(new CBlockTemplate()); if(!pblocktemplate.get()) return NULL; CBlock *pblock = &pblocktemplate->block; // pointer for convenience diff --git a/src/rpcmining.cpp b/src/rpcmining.cpp index b62b29e9a..e4273e42b 100644 --- a/src/rpcmining.cpp +++ b/src/rpcmining.cpp @@ -206,9 +206,9 @@ UniValue generate(const UniValue& params, bool fHelp) while (nHeight < nHeightEnd) { #ifdef ENABLE_WALLET - unique_ptr pblocktemplate(CreateNewBlockWithKey(reservekey)); + std::unique_ptr pblocktemplate(CreateNewBlockWithKey(reservekey)); #else - unique_ptr pblocktemplate(CreateNewBlockWithKey()); + std::unique_ptr pblocktemplate(CreateNewBlockWithKey()); #endif if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Wallet keypool empty"); From 6415573a590ad38ff06540da0b23af3f2f195084 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 14 Sep 2016 15:08:34 +0200 Subject: [PATCH 24/41] bitcoin-cli: More detailed error reporting Register a evhttp error handler to get a more detailed error message if the HTTP request fails. --- src/bitcoin-cli.cpp | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 7cad3f3b4..ea3183dc0 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -112,17 +112,42 @@ static bool AppInitRPC(int argc, char* argv[]) /** Reply structure for request_done to fill in */ struct HTTPReply { + HTTPReply(): status(0), error(-1) {} + int status; + int error; std::string body; }; +const char *http_errorstring(int code) +{ + switch(code) { +#if LIBEVENT_VERSION_NUMBER >= 0x02010300 + case EVREQ_HTTP_TIMEOUT: + return "timeout reached"; + case EVREQ_HTTP_EOF: + return "EOF reached"; + case EVREQ_HTTP_INVALID_HEADER: + return "error while reading header, or invalid header"; + case EVREQ_HTTP_BUFFER_ERROR: + return "error encountered while reading or writing"; + case EVREQ_HTTP_REQUEST_CANCEL: + return "request was canceled"; + case EVREQ_HTTP_DATA_TOO_LONG: + return "response body is larger than allowed"; +#endif + default: + return "unknown"; + } +} + static void http_request_done(struct evhttp_request *req, void *ctx) { HTTPReply *reply = static_cast(ctx); if (req == NULL) { - /* If req is NULL, it means an error occurred while connecting, but - * I'm not sure how to find out which one. We also don't really care. + /* If req is NULL, it means an error occurred while connecting: the + * error code will have been passed to http_error_cb. */ reply->status = 0; return; @@ -141,6 +166,14 @@ static void http_request_done(struct evhttp_request *req, void *ctx) } } +#if LIBEVENT_VERSION_NUMBER >= 0x02010300 +static void http_error_cb(enum evhttp_request_error err, void *ctx) +{ + HTTPReply *reply = static_cast(ctx); + reply->error = err; +} +#endif + UniValue CallRPC(const string& strMethod, const UniValue& params) { std::string host = GetArg("-rpcconnect", "127.0.0.1"); @@ -161,6 +194,9 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) struct evhttp_request *req = evhttp_request_new(http_request_done, (void*)&response); // TODO RAII if (req == NULL) throw runtime_error("create http request failed"); +#if LIBEVENT_VERSION_NUMBER >= 0x02010300 + evhttp_request_set_error_cb(req, http_error_cb); +#endif // Get credentials std::string strRPCUserColonPass; @@ -200,7 +236,7 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) event_base_free(base); if (response.status == 0) - throw CConnectionFailed("couldn't connect to server"); + throw CConnectionFailed(strprintf("couldn't connect to server (%d %s)", response.error, http_errorstring(response.error))); else if (response.status == HTTP_UNAUTHORIZED) throw runtime_error("incorrect rpcuser or rpcpassword (authorization failed)"); else if (response.status >= 400 && response.status != HTTP_BAD_REQUEST && response.status != HTTP_NOT_FOUND && response.status != HTTP_INTERNAL_SERVER_ERROR) From 32f4e7744c11318672dd0e23cef76de0ca388252 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Wed, 14 Sep 2016 17:25:53 +0000 Subject: [PATCH 25/41] depends: Add libevent compatibility patch for windows Add a patch that seems to be necessary for compatibilty of libevent 2.0.22 with recent mingw-w64 gcc versions (at least GCC 5.3.1 from Ubuntu 16.04). Without this patch the Content-Length in the HTTP header ends up as `Content-Length: zu`, causing communication between the RPC client and server to break down. See discussion in #8653. Source: https://sourceforge.net/p/levent/bugs/363/ Thanks to @sstone for the suggestion. --- depends/packages/libevent.mk | 5 +++-- .../patches/libevent/libevent-2-fixes.patch | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 depends/patches/libevent/libevent-2-fixes.patch diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 2e9be1e98..4b02b2eff 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -3,10 +3,11 @@ $(package)_version=2.0.22 $(package)_download_path=https://github.com/libevent/libevent/releases/download/release-2.0.22-stable $(package)_file_name=$(package)-$($(package)_version)-stable.tar.gz $(package)_sha256_hash=71c2c49f0adadacfdbe6332a372c38cf9c8b7895bb73dabeaa53cdcc1d4e1fa3 -$(package)_patches=reuseaddr.patch +$(package)_patches=reuseaddr.patch libevent-2-fixes.patch define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/reuseaddr.patch + patch -p1 < $($(package)_patch_dir)/reuseaddr.patch && \ + patch -p1 < $($(package)_patch_dir)/libevent-2-fixes.patch endef define $(package)_set_vars diff --git a/depends/patches/libevent/libevent-2-fixes.patch b/depends/patches/libevent/libevent-2-fixes.patch new file mode 100644 index 000000000..79fec8a48 --- /dev/null +++ b/depends/patches/libevent/libevent-2-fixes.patch @@ -0,0 +1,18 @@ +--- a/util-internal.h 2013-11-01 12:18:57.000000000 -0600 ++++ b/util-internal.h 2015-07-20 20:19:43.199560900 -0500 +@@ -299,8 +299,13 @@ HANDLE evutil_load_windows_system_librar + + #if defined(__STDC__) && defined(__STDC_VERSION__) + #if (__STDC_VERSION__ >= 199901L) +-#define EV_SIZE_FMT "%zu" +-#define EV_SSIZE_FMT "%zd" ++ #if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__) ++ #define EV_SIZE_FMT "%Iu" ++ #define EV_SSIZE_FMT "%Id" ++ #else ++ #define EV_SIZE_FMT "%zu" ++ #define EV_SSIZE_FMT "%zd" ++ #endif + #define EV_SIZE_ARG(x) (x) + #define EV_SSIZE_ARG(x) (x) + #endif From 603205e3813c4530bc60984790fba1ee28695157 Mon Sep 17 00:00:00 2001 From: instagibbs Date: Thu, 3 Nov 2016 09:56:26 -0400 Subject: [PATCH 26/41] Add common failure cases for rpc server connection failure --- src/bitcoin-cli.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index ea3183dc0..c5e0d1ad6 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -236,7 +236,7 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) event_base_free(base); if (response.status == 0) - throw CConnectionFailed(strprintf("couldn't connect to server (%d %s)", response.error, http_errorstring(response.error))); + throw CConnectionFailed(strprintf("couldn't connect to server\n(make sure server is running and you are connecting to the correct RPC port: %d %s)", response.error, http_errorstring(response.error))); else if (response.status == HTTP_UNAUTHORIZED) throw runtime_error("incorrect rpcuser or rpcpassword (authorization failed)"); else if (response.status >= 400 && response.status != HTTP_BAD_REQUEST && response.status != HTTP_NOT_FOUND && response.status != HTTP_INTERNAL_SERVER_ERROR) From 7c2ab05969059a1b4b6a8f19455e7d4438a47b78 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Fri, 2 Dec 2016 08:51:36 +0100 Subject: [PATCH 27/41] bitcoin-cli: Make error message less confusing Sorry for the churn on this, but the current message (introduced in #9073) isn't acceptable: $ src/bitcoin-cli getinfo rpc: couldn't connect to server (make sure server is running and you are connecting to the correct RPC port: -1 unknown) Putting the error code after the words "RPC port" made me wonder whether there was a port configuration issue. This changes it to: $ src/bitcoin-cli getinfo error: couldn't connect to server: unknown (code -1) (make sure server is running and you are connecting to the correct RPC port) --- src/bitcoin-cli.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index c5e0d1ad6..426c02477 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -236,7 +236,7 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) event_base_free(base); if (response.status == 0) - throw CConnectionFailed(strprintf("couldn't connect to server\n(make sure server is running and you are connecting to the correct RPC port: %d %s)", response.error, http_errorstring(response.error))); + throw CConnectionFailed(strprintf("couldn't connect to server: %s (code %d)\n(make sure server is running and you are connecting to the correct RPC port)", http_errorstring(response.error), response.error)); else if (response.status == HTTP_UNAUTHORIZED) throw runtime_error("incorrect rpcuser or rpcpassword (authorization failed)"); else if (response.status >= 400 && response.status != HTTP_BAD_REQUEST && response.status != HTTP_NOT_FOUND && response.status != HTTP_INTERNAL_SERVER_ERROR) From 17fb609097b408cbba9b5d054467e6fc2c2e2d2f Mon Sep 17 00:00:00 2001 From: Karl-Johan Alm Date: Tue, 20 Dec 2016 15:59:42 +0900 Subject: [PATCH 28/41] Added std::unique_ptr<> wrappers with deleters for libevent modules. --- src/Makefile.am | 1 + src/support/events.h | 55 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 src/support/events.h diff --git a/src/Makefile.am b/src/Makefile.am index 75ec72af6..33c1fba2d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -158,6 +158,7 @@ BITCOIN_CORE_H = \ support/allocators/secure.h \ support/allocators/zeroafterfree.h \ support/cleanse.h \ + support/events.h \ support/pagelocker.h \ sync.h \ threadsafety.h \ diff --git a/src/support/events.h b/src/support/events.h new file mode 100644 index 000000000..16378086f --- /dev/null +++ b/src/support/events.h @@ -0,0 +1,55 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SUPPORT_EVENTS_H +#define BITCOIN_SUPPORT_EVENTS_H + +#include + +#include +#include + +#define MAKE_RAII(type) \ +/* deleter */\ +struct type##_deleter {\ + void operator()(struct type* ob) {\ + type##_free(ob);\ + }\ +};\ +/* unique ptr typedef */\ +typedef std::unique_ptr raii_##type + +MAKE_RAII(event_base); +MAKE_RAII(event); +MAKE_RAII(evhttp); +MAKE_RAII(evhttp_request); +MAKE_RAII(evhttp_connection); + +raii_event_base obtain_event_base() { + auto result = raii_event_base(event_base_new()); + if (!result.get()) + throw std::runtime_error("cannot create event_base"); + return result; +} + +raii_event obtain_event(struct event_base* base, evutil_socket_t s, short events, event_callback_fn cb, void* arg) { + return raii_event(event_new(base, s, events, cb, arg)); +} + +raii_evhttp obtain_evhttp(struct event_base* base) { + return raii_evhttp(evhttp_new(base)); +} + +raii_evhttp_request obtain_evhttp_request(void(*cb)(struct evhttp_request *, void *), void *arg) { + return raii_evhttp_request(evhttp_request_new(cb, arg)); +} + +raii_evhttp_connection obtain_evhttp_connection_base(struct event_base* base, std::string host, uint16_t port) { + auto result = raii_evhttp_connection(evhttp_connection_base_new(base, NULL, host.c_str(), port)); + if (!result.get()) + throw std::runtime_error("create connection failed"); + return result; +} + +#endif // BITCOIN_SUPPORT_EVENTS_H From 68377e18a81f9d285b455e4aca7557043beca09a Mon Sep 17 00:00:00 2001 From: Karl-Johan Alm Date: Tue, 20 Dec 2016 20:46:11 +0900 Subject: [PATCH 29/41] Switched bitcoin-cli.cpp to use RAII unique pointers with deleters. --- src/bitcoin-cli.cpp | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 426c02477..46c9f5fc4 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -13,10 +13,9 @@ #include #include -#include -#include #include #include +#include "support/events.h" #include @@ -179,23 +178,19 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) std::string host = GetArg("-rpcconnect", "127.0.0.1"); int port = GetArg("-rpcport", BaseParams().RPCPort()); - // Create event base - struct event_base *base = event_base_new(); // TODO RAII - if (!base) - throw runtime_error("cannot create event_base"); + // Obtain event base + raii_event_base base = obtain_event_base(); // Synchronously look up hostname - struct evhttp_connection *evcon = evhttp_connection_base_new(base, NULL, host.c_str(), port); // TODO RAII - if (evcon == NULL) - throw runtime_error("create connection failed"); - evhttp_connection_set_timeout(evcon, GetArg("-rpcclienttimeout", DEFAULT_HTTP_CLIENT_TIMEOUT)); + raii_evhttp_connection evcon = obtain_evhttp_connection_base(base.get(), host, port); + evhttp_connection_set_timeout(evcon.get(), GetArg("-rpcclienttimeout", DEFAULT_HTTP_CLIENT_TIMEOUT)); HTTPReply response; - struct evhttp_request *req = evhttp_request_new(http_request_done, (void*)&response); // TODO RAII + raii_evhttp_request req = obtain_evhttp_request(http_request_done, (void*)&response); if (req == NULL) throw runtime_error("create http request failed"); #if LIBEVENT_VERSION_NUMBER >= 0x02010300 - evhttp_request_set_error_cb(req, http_error_cb); + evhttp_request_set_error_cb(req.get(), http_error_cb); #endif // Get credentials @@ -212,7 +207,7 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) strRPCUserColonPass = mapArgs["-rpcuser"] + ":" + mapArgs["-rpcpassword"]; } - struct evkeyvalq *output_headers = evhttp_request_get_output_headers(req); + struct evkeyvalq* output_headers = evhttp_request_get_output_headers(req.get()); assert(output_headers); evhttp_add_header(output_headers, "Host", host.c_str()); evhttp_add_header(output_headers, "Connection", "close"); @@ -220,20 +215,17 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) // Attach request data std::string strRequest = JSONRPCRequest(strMethod, params, 1); - struct evbuffer * output_buffer = evhttp_request_get_output_buffer(req); + struct evbuffer* output_buffer = evhttp_request_get_output_buffer(req.get()); assert(output_buffer); evbuffer_add(output_buffer, strRequest.data(), strRequest.size()); - int r = evhttp_make_request(evcon, req, EVHTTP_REQ_POST, "/"); + int r = evhttp_make_request(evcon.get(), req.get(), EVHTTP_REQ_POST, "/"); + req.release(); // ownership moved to evcon in above call if (r != 0) { - evhttp_connection_free(evcon); - event_base_free(base); throw CConnectionFailed("send http request failed"); } - event_base_dispatch(base); - evhttp_connection_free(evcon); - event_base_free(base); + event_base_dispatch(base.get()); if (response.status == 0) throw CConnectionFailed(strprintf("couldn't connect to server: %s (code %d)\n(make sure server is running and you are connecting to the correct RPC port)", http_errorstring(response.error), response.error)); From e8b6d84b9f1607798dc25fc9b5404fff18c8d85e Mon Sep 17 00:00:00 2001 From: Karl-Johan Alm Date: Wed, 21 Dec 2016 13:43:49 +0900 Subject: [PATCH 30/41] Added some simple tests for the RAII-style events. --- src/Makefile.test.include | 3 +- src/support/events.h | 1 + src/test/raii_event_tests.cpp | 88 +++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 src/test/raii_event_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 4defc09ed..25c226c58 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -69,6 +69,7 @@ BITCOIN_TESTS =\ test/pmt_tests.cpp \ test/policyestimator_tests.cpp \ test/pow_tests.cpp \ + test/raii_event_tests.cpp \ test/reverselock_tests.cpp \ test/rpc_tests.cpp \ test/sanity_tests.cpp \ @@ -99,7 +100,7 @@ endif test_test_bitcoin_SOURCES = $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) test_test_bitcoin_CPPFLAGS = -fopenmp $(BITCOIN_INCLUDES) -I$(builddir)/test/ $(TESTDEFS) test_test_bitcoin_LDADD = $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ - $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) + $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) if ENABLE_WALLET test_test_bitcoin_LDADD += $(LIBBITCOIN_WALLET) endif diff --git a/src/support/events.h b/src/support/events.h index 16378086f..4f2f3cf9e 100644 --- a/src/support/events.h +++ b/src/support/events.h @@ -6,6 +6,7 @@ #define BITCOIN_SUPPORT_EVENTS_H #include +#include #include #include diff --git a/src/test/raii_event_tests.cpp b/src/test/raii_event_tests.cpp new file mode 100644 index 000000000..87d25c0e2 --- /dev/null +++ b/src/test/raii_event_tests.cpp @@ -0,0 +1,88 @@ +// Copyright (c) 2016 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include + +#include "support/events.h" + +#include "test/test_bitcoin.h" + +#include + +#include + +static std::map tags; +static std::map orders; +static uint16_t tagSequence = 0; + +static void* tag_malloc(size_t sz) { + void* mem = malloc(sz); + if (!mem) return mem; + tags[mem]++; + orders[mem] = tagSequence++; + return mem; +} + +static void tag_free(void* mem) { + tags[mem]--; + orders[mem] = tagSequence++; + free(mem); +} + +BOOST_FIXTURE_TEST_SUITE(raii_event_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(raii_event_creation) +{ + event_set_mem_functions(tag_malloc, realloc, tag_free); + + void* base_ptr = NULL; + { + auto base = obtain_event_base(); + base_ptr = (void*)base.get(); + BOOST_CHECK(tags[base_ptr] == 1); + } + BOOST_CHECK(tags[base_ptr] == 0); + + void* event_ptr = NULL; + { + auto base = obtain_event_base(); + auto event = obtain_event(base.get(), -1, 0, NULL, NULL); + + base_ptr = (void*)base.get(); + event_ptr = (void*)event.get(); + + BOOST_CHECK(tags[base_ptr] == 1); + BOOST_CHECK(tags[event_ptr] == 1); + } + BOOST_CHECK(tags[base_ptr] == 0); + BOOST_CHECK(tags[event_ptr] == 0); + + event_set_mem_functions(malloc, realloc, free); +} + +BOOST_AUTO_TEST_CASE(raii_event_order) +{ + event_set_mem_functions(tag_malloc, realloc, tag_free); + + void* base_ptr = NULL; + void* event_ptr = NULL; + { + auto base = obtain_event_base(); + auto event = obtain_event(base.get(), -1, 0, NULL, NULL); + + base_ptr = (void*)base.get(); + event_ptr = (void*)event.get(); + + // base should have allocated before event + BOOST_CHECK(orders[base_ptr] < orders[event_ptr]); + } + // base should be freed after event + BOOST_CHECK(orders[base_ptr] > orders[event_ptr]); + + event_set_mem_functions(malloc, realloc, free); +} + +BOOST_AUTO_TEST_SUITE_END() From ca50af75b10c65d0dc78cbf86ca19f89fc4d5892 Mon Sep 17 00:00:00 2001 From: Karl-Johan Alm Date: Wed, 4 Jan 2017 18:16:47 +0900 Subject: [PATCH 31/41] Added EVENT_CFLAGS to test makefile to explicitly include libevent headers. --- src/Makefile.test.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 25c226c58..7de8c392f 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -98,7 +98,7 @@ BITCOIN_TESTS += \ endif test_test_bitcoin_SOURCES = $(BITCOIN_TESTS) $(JSON_TEST_FILES) $(RAW_TEST_FILES) -test_test_bitcoin_CPPFLAGS = -fopenmp $(BITCOIN_INCLUDES) -I$(builddir)/test/ $(TESTDEFS) +test_test_bitcoin_CPPFLAGS = -fopenmp $(BITCOIN_INCLUDES) -I$(builddir)/test/ $(TESTDEFS) $(EVENT_CFLAGS) test_test_bitcoin_LDADD = $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) \ $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB) $(LIBSECP256K1) $(EVENT_LIBS) if ENABLE_WALLET From ec626cc530f40a0e9f3bb7f1119d40da00637548 Mon Sep 17 00:00:00 2001 From: fanquake Date: Wed, 4 Jan 2017 22:00:26 +0800 Subject: [PATCH 32/41] [depends] libevent 2.1.7rc --- depends/packages/libevent.mk | 12 +++++------ .../patches/libevent/libevent-2-fixes.patch | 18 ---------------- depends/patches/libevent/reuseaddr.patch | 21 ------------------- 3 files changed, 5 insertions(+), 46 deletions(-) delete mode 100644 depends/patches/libevent/libevent-2-fixes.patch delete mode 100644 depends/patches/libevent/reuseaddr.patch diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 4b02b2eff..70f345e71 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -1,13 +1,11 @@ package=libevent -$(package)_version=2.0.22 -$(package)_download_path=https://github.com/libevent/libevent/releases/download/release-2.0.22-stable -$(package)_file_name=$(package)-$($(package)_version)-stable.tar.gz -$(package)_sha256_hash=71c2c49f0adadacfdbe6332a372c38cf9c8b7895bb73dabeaa53cdcc1d4e1fa3 -$(package)_patches=reuseaddr.patch libevent-2-fixes.patch +$(package)_version=2.1.7 +$(package)_download_path=https://github.com/libevent/libevent/archive/ +$(package)_file_name=release-$($(package)_version)-rc.tar.gz +$(package)_sha256_hash=548362d202e22fe24d4c3fad38287b4f6d683e6c21503341373b89785fa6f991 define $(package)_preprocess_cmds - patch -p1 < $($(package)_patch_dir)/reuseaddr.patch && \ - patch -p1 < $($(package)_patch_dir)/libevent-2-fixes.patch + ./autogen.sh endef define $(package)_set_vars diff --git a/depends/patches/libevent/libevent-2-fixes.patch b/depends/patches/libevent/libevent-2-fixes.patch deleted file mode 100644 index 79fec8a48..000000000 --- a/depends/patches/libevent/libevent-2-fixes.patch +++ /dev/null @@ -1,18 +0,0 @@ ---- a/util-internal.h 2013-11-01 12:18:57.000000000 -0600 -+++ b/util-internal.h 2015-07-20 20:19:43.199560900 -0500 -@@ -299,8 +299,13 @@ HANDLE evutil_load_windows_system_librar - - #if defined(__STDC__) && defined(__STDC_VERSION__) - #if (__STDC_VERSION__ >= 199901L) --#define EV_SIZE_FMT "%zu" --#define EV_SSIZE_FMT "%zd" -+ #if defined(_MSC_VER) || defined(__MINGW32__) || defined(__MINGW64__) -+ #define EV_SIZE_FMT "%Iu" -+ #define EV_SSIZE_FMT "%Id" -+ #else -+ #define EV_SIZE_FMT "%zu" -+ #define EV_SSIZE_FMT "%zd" -+ #endif - #define EV_SIZE_ARG(x) (x) - #define EV_SSIZE_ARG(x) (x) - #endif diff --git a/depends/patches/libevent/reuseaddr.patch b/depends/patches/libevent/reuseaddr.patch deleted file mode 100644 index 58695c11f..000000000 --- a/depends/patches/libevent/reuseaddr.patch +++ /dev/null @@ -1,21 +0,0 @@ ---- old/evutil.c 2015-08-28 19:26:23.488765923 -0400 -+++ new/evutil.c 2015-08-28 19:27:41.392767019 -0400 -@@ -321,15 +321,16 @@ - int - evutil_make_listen_socket_reuseable(evutil_socket_t sock) - { --#ifndef WIN32 - int one = 1; -+#ifndef WIN32 - /* REUSEADDR on Unix means, "don't hang on to this address after the - * listener is closed." On Windows, though, it means "don't keep other - * processes from binding to this address while we're using it. */ - return setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (void*) &one, - (ev_socklen_t)sizeof(one)); - #else -- return 0; -+ return setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, (const char*) &one, -+ (ev_socklen_t)sizeof(one)); - #endif - } - From c65969a5a693f5dbbcb167047cee984c24871a52 Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Sat, 28 Jan 2017 08:03:57 +0000 Subject: [PATCH 33/41] Skip RAII event tests if libevent is built without event_set_mem_functions --- src/test/raii_event_tests.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test/raii_event_tests.cpp b/src/test/raii_event_tests.cpp index 87d25c0e2..0f40874f5 100644 --- a/src/test/raii_event_tests.cpp +++ b/src/test/raii_event_tests.cpp @@ -3,6 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include + +#ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED +// It would probably be ideal to define dummy test(s) that report skipped, but boost::test doesn't seem to make that practical (at least not in versions available with common distros) + #include #include @@ -86,3 +90,5 @@ BOOST_AUTO_TEST_CASE(raii_event_order) } BOOST_AUTO_TEST_SUITE_END() + +#endif // EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED From 629a87522917b66541011fbf794fced04d104df7 Mon Sep 17 00:00:00 2001 From: Ian Kelling Date: Thu, 2 Mar 2017 04:42:37 -0800 Subject: [PATCH 34/41] Docs: add details to -rpcclienttimeout doc --- src/bitcoin-cli.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 46c9f5fc4..81366205f 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -38,7 +38,7 @@ std::string HelpMessageCli() strUsage += HelpMessageOpt("-rpcwait", _("Wait for RPC server to start")); strUsage += HelpMessageOpt("-rpcuser=", _("Username for JSON-RPC connections")); strUsage += HelpMessageOpt("-rpcpassword=", _("Password for JSON-RPC connections")); - strUsage += HelpMessageOpt("-rpcclienttimeout=", strprintf(_("Timeout during HTTP requests (default: %d)"), DEFAULT_HTTP_CLIENT_TIMEOUT)); + strUsage += HelpMessageOpt("-rpcclienttimeout=", strprintf(_("Timeout in seconds during HTTP requests, or 0 for no timeout. (default: %d)"), DEFAULT_HTTP_CLIENT_TIMEOUT)); return strUsage; } From a163953929830f509308f8bfc0fc207eab398420 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Thu, 23 Mar 2017 10:34:11 +1300 Subject: [PATCH 35/41] [depends] libevent 2.1.8 --- depends/packages/libevent.mk | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/depends/packages/libevent.mk b/depends/packages/libevent.mk index 70f345e71..2a1125fda 100644 --- a/depends/packages/libevent.mk +++ b/depends/packages/libevent.mk @@ -1,8 +1,9 @@ package=libevent -$(package)_version=2.1.7 +$(package)_version=2.1.8 $(package)_download_path=https://github.com/libevent/libevent/archive/ -$(package)_file_name=release-$($(package)_version)-rc.tar.gz -$(package)_sha256_hash=548362d202e22fe24d4c3fad38287b4f6d683e6c21503341373b89785fa6f991 +$(package)_file_name=$(package)-$($(package)_version).tar.gz +$(package)_download_file=release-$($(package)_version)-stable.tar.gz +$(package)_sha256_hash=316ddb401745ac5d222d7c529ef1eada12f58f6376a66c1118eee803cb70f83d define $(package)_preprocess_cmds ./autogen.sh From 17694e4bcf29904d2bdb6159ad5c40e17383ab6d Mon Sep 17 00:00:00 2001 From: paveljanik Date: Fri, 4 Sep 2015 19:22:48 +0200 Subject: [PATCH 36/41] [TRIVIAL] Fix typo: exactmath -> exactmatch ... but not yet in trivial tree --- src/httpserver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/httpserver.cpp b/src/httpserver.cpp index c04c9da56..4215a0f26 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -578,7 +578,7 @@ HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod() void RegisterHTTPHandler(const std::string &prefix, bool exactMatch, const HTTPRequestHandler &handler) { - LogPrint("http", "Registering HTTP handler for %s (exactmath %d)\n", prefix, exactMatch); + LogPrint("http", "Registering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); pathHandlers.push_back(HTTPPathHandler(prefix, exactMatch, handler)); } @@ -591,7 +591,7 @@ void UnregisterHTTPHandler(const std::string &prefix, bool exactMatch) break; if (i != iend) { - LogPrint("http", "Unregistering HTTP handler for %s (exactmath %d)\n", prefix, exactMatch); + LogPrint("http", "Unregistering HTTP handler for %s (exactmatch %d)\n", prefix, exactMatch); pathHandlers.erase(i); } } From 2edad86b2d130fa424ce63220dbf536c3efa9b55 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Fri, 13 May 2016 04:49:19 +0200 Subject: [PATCH 37/41] Fix interrupted HTTP RPC connection workaround for Python 3.5+ --- qa/rpc-tests/test_framework/authproxy.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index 111e6d76c..5df359c63 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -125,6 +125,11 @@ class AuthServiceProxy(object): return self._get_response() else: raise + except BrokenPipeError: + # Python 3.5+ raises this instead of BadStatusLine when the connection was reset + self.__conn.close() + self.__conn.request(method, path, postdata, headers) + return self._get_response() def __call__(self, *args): AuthServiceProxy.__id_count += 1 From e93a3e19456a26c30ec048deececd4d0cd9ca4eb Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 29 Sep 2016 15:02:08 +0000 Subject: [PATCH 38/41] test: Avoid ConnectionResetErrors during RPC tests This is necessary on FreeBSD and MacOSX, at least. See https://github.com/bitcoin/bitcoin/pull/8834#issuecomment-250450213 --- qa/rpc-tests/test_framework/authproxy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/test_framework/authproxy.py b/qa/rpc-tests/test_framework/authproxy.py index 5df359c63..ad0a31798 100644 --- a/qa/rpc-tests/test_framework/authproxy.py +++ b/qa/rpc-tests/test_framework/authproxy.py @@ -125,8 +125,9 @@ class AuthServiceProxy(object): return self._get_response() else: raise - except BrokenPipeError: - # Python 3.5+ raises this instead of BadStatusLine when the connection was reset + except (BrokenPipeError,ConnectionResetError): + # Python 3.5+ raises BrokenPipeError instead of BadStatusLine when the connection was reset + # ConnectionResetError happens on FreeBSD with Python 3.4 self.__conn.close() self.__conn.request(method, path, postdata, headers) return self._get_response() From 47855b599d415cf2c4a2f29dca9ee5f9589d3744 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 25 Mar 2017 17:37:18 +1300 Subject: [PATCH 39/41] Revert "Revert "rpc-tests: re-enable rpc-tests for Windows"" This reverts commit 7fd5d4e30e3d01e301c083987886ffd86f2df7d7. We need to un-revert this before pulling in https://github.com/bitcoin/bitcoin/pull/6616 --- qa/pull-tester/rpc-tests.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/qa/pull-tester/rpc-tests.sh b/qa/pull-tester/rpc-tests.sh index 4650e1d52..08ff3fe7a 100755 --- a/qa/pull-tester/rpc-tests.sh +++ b/qa/pull-tester/rpc-tests.sh @@ -8,11 +8,6 @@ CURDIR=$(cd $(dirname "$0"); pwd) export BITCOINCLI=${BUILDDIR}/qa/pull-tester/run-bitcoin-cli export BITCOIND=${REAL_BITCOIND} -if [ "x${EXEEXT}" = "x.exe" ]; then - echo "Win tests currently disabled" - exit 0 -fi - #Run the tests testScripts=( From 206e2b97373664c6853d030b2036ec3d9aebfaff Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 25 Mar 2017 17:39:56 +1300 Subject: [PATCH 40/41] Wrap error string --- src/bitcoin-cli.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 81366205f..a4f948bb0 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -199,7 +199,8 @@ UniValue CallRPC(const string& strMethod, const UniValue& params) // Try fall back to cookie-based authentication if no password is provided if (!GetAuthCookie(&strRPCUserColonPass)) { throw runtime_error(strprintf( - _("Could not locate RPC credentials. No authentication cookie could be found, and no rpcpassword is set in the configuration file (%s)"), + _("Could not locate RPC credentials. No authentication cookie could be found,\n" + "and no rpcpassword is set in the configuration file (%s)."), GetConfigFile().string().c_str())); } From 3da13e885e60a3b55450a4381d3d6b0b613a215b Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Sat, 25 Mar 2017 17:40:27 +1300 Subject: [PATCH 41/41] Fix typo --- src/rpcserver.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpcserver.h b/src/rpcserver.h index 56fe2542e..1124801c2 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -101,7 +101,7 @@ public: * RPC will call the function to create a timer that will call func in *millis* milliseconds. * @note As the RPC mechanism is backend-neutral, it can use different implementations of timers. * This is needed to cope with the case in which there is no HTTP server, but - * only GUI RPC console, and to break the dependency of pcserver on httprpc. + * only GUI RPC console, and to break the dependency of rpcserver on httprpc. */ virtual RPCTimerBase* NewTimer(boost::function& func, int64_t millis) = 0; };