feature(rpc): add an rpc server to Zebra (#3589)

* feature(rpc): add an rpc component

* feat(rpc): add a stub for getblockchaininfo

This is the first RPC used by lightwalletd, so we need it for testing.

* fix(rpc): remove non-standard "jsonrpc: 1.0" from lightwalletd

* fix(rpc): re-enable default RPC security checks

* deps(rpc): remove not needed dependency

* fix(rpc): check if RPC task has stopped

* fix(rpc): reduce config by using Option

* fix(rpc): use tokio executor

* security(rpc): turn off rpc by default

* docs(rpc): update a TODO comment

Co-authored-by: teor <teor@riseup.net>

* fix(rpc): blocking tasks

Co-authored-by: teor <teor@riseup.net>

* rename(rpc): rpc.rs to methods.rs

* refactor(rpc): move the server to the zebra-rpc crate

* fix(rpc): clippy derive Default for RPC Config

* fix(dependencies): remove unused dependency features in zebra-rpc

We expect to use all the listed tokio features
to implement and test RPC methods.

* doc(rpc): fix testnet port, add security note

* fix(rpc): change Rust function names and update method doc TODOs

* fix(rpc): add "TODO" to fake RPC responses

* doc(rpc): update module docs

* fix(rpc): simplify server struct derives

* fix(rpc): simplify server code

* doc(rpc): explain how request fixes securely handle user-supplied data

* refactor(rpc): move the compatibility fix to a separate module

* fix(rpc): move the open log inside the spawn, and instrument it

* doc(rpc): fix toml format and provide a config example

Co-authored-by: teor <teor@riseup.net>

Cargo.lock

@@ -1655,6 +1655,19 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 
+[[package]]
+name = "globset"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10463d9ff00a2a068db14231982f5132edebad0d7660cd956a1c30292dbcbfbd"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "fnv",
+ "log",
+ "regex",
+]
+
 [[package]]
 name = "group"
 version = "0.11.0"
@@ -2021,6 +2034,67 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "jsonrpc-core"
+version = "18.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb"
+dependencies = [
+ "futures",
+ "futures-executor",
+ "futures-util",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+]
+
+[[package]]
+name = "jsonrpc-derive"
+version = "18.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2 1.0.34",
+ "quote 1.0.10",
+ "syn 1.0.83",
+]
+
+[[package]]
+name = "jsonrpc-http-server"
+version = "18.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1dea6e07251d9ce6a552abfb5d7ad6bc290a4596c8dcc3d795fae2bbdc1f3ff"
+dependencies = [
+ "futures",
+ "hyper",
+ "jsonrpc-core",
+ "jsonrpc-server-utils",
+ "log",
+ "net2",
+ "parking_lot",
+ "unicase",
+]
+
+[[package]]
+name = "jsonrpc-server-utils"
+version = "18.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4fdea130485b572c39a460d50888beb00afb3e35de23ccd7fad8ff19f0e0d4"
+dependencies = [
+ "bytes",
+ "futures",
+ "globset",
+ "jsonrpc-core",
+ "lazy_static",
+ "log",
+ "tokio",
+ "tokio-stream",
+ "tokio-util 0.6.9",
+ "unicase",
+]
+
 [[package]]
 name = "jubjub"
 version = "0.8.0"
@@ -2401,6 +2475,17 @@ dependencies = [
  "tempfile",
 ]
 
+[[package]]
+name = "net2"
+version = "0.2.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae"
+dependencies = [
+ "cfg-if 0.1.10",
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "nibble_vec"
 version = "0.1.0"
@@ -2931,6 +3016,15 @@ version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857"
 
+[[package]]
+name = "proc-macro-crate"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785"
+dependencies = [
+ "toml",
+]
+
 [[package]]
 name = "proc-macro-error"
 version = "1.0.4"
@@ -4979,6 +5073,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "unicase"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
+dependencies = [
+ "version_check",
+]
+
 [[package]]
 name = "unicode-bidi"
 version = "0.3.4"
@@ -5588,6 +5691,17 @@ dependencies = [
 [[package]]
 name = "zebra-rpc"
 version = "1.0.0-beta.0"
+dependencies = [
+ "futures",
+ "hyper",
+ "jsonrpc-core",
+ "jsonrpc-derive",
+ "jsonrpc-http-server",
+ "serde",
+ "tokio",
+ "tracing",
+ "tracing-futures",
+]
 
 [[package]]
 name = "zebra-script"
@@ -5717,6 +5831,7 @@ dependencies = [
  "zebra-chain",
  "zebra-consensus",
  "zebra-network",
+ "zebra-rpc",
  "zebra-state",
  "zebra-test",
 ]

zebra-rpc/Cargo.toml

@@ -8,3 +8,19 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+
+futures = "0.3"
+
+# lightwalletd sends JSON-RPC requests over HTTP 1.1
+hyper = { version = "0.14.17", features = ["http1", "server"] }
+
+jsonrpc-core = "18.0.0"
+jsonrpc-derive = "18.0.0"
+jsonrpc-http-server = "18.0.0"
+
+tokio = { version = "1.16.1", features = ["time", "rt-multi-thread", "macros", "tracing"] }
+
+tracing = "0.1"
+tracing-futures = "0.2"
+
+serde = { version = "1", features = ["serde_derive"] }

zebra-rpc/src/config.rs

@@ -0,0 +1,30 @@
+//! User-configurable RPC settings.
+
+use std::net::SocketAddr;
+
+use serde::{Deserialize, Serialize};
+
+/// RPC configuration section.
+#[derive(Clone, Debug, Default, Deserialize, Serialize)]
+#[serde(deny_unknown_fields, default)]
+pub struct Config {
+    /// IP address and port for the RPC server.
+    ///
+    /// Note: The RPC server is disabled by default.
+    /// To enable the RPC server, set a listen address in the config:
+    /// ```toml
+    /// [rpc]
+    /// listen_addr = '127.0.0.1:8232'
+    /// ```
+    ///
+    /// The recommended ports for the RPC server are:
+    /// - Mainnet: 127.0.0.1:8232
+    /// - Testnet: 127.0.0.1:18232
+    ///
+    /// # Security
+    ///
+    /// If you bind Zebra's RPC port to a public IP address,
+    /// anyone on the internet can send transactions via your node.
+    /// They can also query your node's state.
+    pub listen_addr: Option<SocketAddr>,
+}

zebra-rpc/src/lib.rs

@@ -1,5 +1,9 @@
-//! A Zebra remote procedure call interface
+//! A Zebra Remote Procedure Call (RPC) interface
 
 #![doc(html_favicon_url = "https://www.zfnd.org/images/zebra-favicon-128.png")]
 #![doc(html_logo_url = "https://www.zfnd.org/images/zebra-icon.png")]
 #![doc(html_root_url = "https://doc.zebra.zfnd.org/zebra_rpc")]
+
+pub mod config;
+pub mod methods;
+pub mod server;

zebra-rpc/src/methods.rs

@@ -0,0 +1,69 @@
+//! Zebra supported RPC methods.
+//!
+//! Based on the [`zcashd` RPC methods](https://zcash.github.io/rpc/)
+//! as used by `lightwalletd.`
+//!
+//! Some parts of the `zcashd` RPC documentation are outdated.
+//! So this implementation follows the `lightwalletd` client implementation.
+
+use jsonrpc_core::{self, Result};
+use jsonrpc_derive::rpc;
+
+#[rpc(server)]
+/// RPC method signatures.
+pub trait Rpc {
+    /// getinfo
+    ///
+    /// TODO: explain what the method does
+    ///       link to the zcashd RPC reference
+    ///       list the arguments and fields that lightwalletd uses
+    ///       note any other lightwalletd changes
+    #[rpc(name = "getinfo")]
+    fn get_info(&self) -> Result<GetInfo>;
+
+    /// getblockchaininfo
+    ///
+    /// TODO: explain what the method does
+    ///       link to the zcashd RPC reference
+    ///       list the arguments and fields that lightwalletd uses
+    ///       note any other lightwalletd changes
+    #[rpc(name = "getblockchaininfo")]
+    fn get_blockchain_info(&self) -> Result<GetBlockChainInfo>;
+}
+
+/// RPC method implementations.
+pub struct RpcImpl;
+impl Rpc for RpcImpl {
+    fn get_info(&self) -> Result<GetInfo> {
+        // TODO: dummy output data, fix in the context of #3142
+        let response = GetInfo {
+            build: "TODO: Zebra v1.0.0 ...".into(),
+            subversion: "TODO: /Zebra:1.0.0-beta.../".into(),
+        };
+
+        Ok(response)
+    }
+
+    fn get_blockchain_info(&self) -> Result<GetBlockChainInfo> {
+        // TODO: dummy output data, fix in the context of #3143
+        let response = GetBlockChainInfo {
+            chain: "TODO: main".to_string(),
+        };
+
+        Ok(response)
+    }
+}
+
+#[derive(serde::Serialize, serde::Deserialize)]
+/// Response to a `getinfo` RPC request.
+pub struct GetInfo {
+    build: String,
+    subversion: String,
+}
+
+#[derive(serde::Serialize, serde::Deserialize)]
+/// Response to a `getblockchaininfo` RPC request.
+pub struct GetBlockChainInfo {
+    chain: String,
+    // TODO: add other fields used by lightwalletd (#3143)
+}

zebra-rpc/src/server.rs

@@ -0,0 +1,63 @@
+//! A JSON-RPC 1.0 & 2.0 endpoint for Zebra.
+//!
+//! This endpoint is compatible with clients that incorrectly send
+//! `"jsonrpc" = 1.0` fields in JSON-RPC 1.0 requests,
+//! such as `lightwalletd`.
+
+use tracing::*;
+use tracing_futures::Instrument;
+
+use jsonrpc_core;
+use jsonrpc_http_server::ServerBuilder;
+
+use crate::{
+    config::Config,
+    methods::{Rpc, RpcImpl},
+    server::compatibility::FixHttpRequestMiddleware,
+};
+
+pub mod compatibility;
+
+/// Zebra RPC Server
+#[derive(Clone, Debug)]
+pub struct RpcServer;
+
+impl RpcServer {
+    /// Start a new RPC server endpoint
+    pub fn spawn(config: Config) -> tokio::task::JoinHandle<()> {
+        if let Some(listen_addr) = config.listen_addr {
+            info!("Trying to open RPC endpoint at {}...", listen_addr,);
+
+            // Create handler compatible with V1 and V2 RPC protocols
+            let mut io =
+                jsonrpc_core::IoHandler::with_compatibility(jsonrpc_core::Compatibility::Both);
+            io.extend_with(RpcImpl.to_delegate());
+
+            let server = ServerBuilder::new(io)
+                // use the same tokio executor as the rest of Zebra
+                .event_loop_executor(tokio::runtime::Handle::current())
+                .threads(1)
+                // TODO: disable this security check if we see errors from lightwalletd.
+                //.allowed_hosts(DomainsValidation::Disabled)
+                .request_middleware(FixHttpRequestMiddleware)
+                .start_http(&listen_addr)
+                .expect("Unable to start RPC server");
+
+            // The server is a blocking task, so we need to spawn it on a blocking thread.
+            let span = Span::current();
+            let server = move || {
+                span.in_scope(|| {
+                    info!("Opened RPC endpoint at {}", server.address());
+
+                    server.wait();
+
+                    info!("Stopping RPC endpoint");
+                })
+            };
+            tokio::task::spawn_blocking(server)
+        } else {
+            // There is no RPC port, so the RPC task does nothing.
+            tokio::task::spawn(futures::future::pending().in_current_span())
+        }
+    }
+}

zebra-rpc/src/server/compatibility.rs

@@ -0,0 +1,85 @@
+//! Compatibility fixes for JSON-RPC requests.
+
+use futures::TryStreamExt;
+use hyper::{body::Bytes, Body};
+
+use jsonrpc_http_server::RequestMiddleware;
+
+/// HTTP [`RequestMiddleware`] with compatibility workarounds.
+///
+/// This middleware makes the following changes to requests:
+///
+/// ## JSON RPC 1.0 `jsonrpc` field
+///
+/// Removes "jsonrpc: 1.0" fields from requests,
+/// because the "jsonrpc" field was only added in JSON-RPC 2.0.
+///
+/// <http://www.simple-is-better.org/rpc/#differences-between-1-0-and-2-0>
+///
+/// ## Security
+///
+/// Any user-specified data in RPC requests is hex or base58check encoded.
+/// We assume lightwalletd validates data encodings before sending it on to Zebra.
+/// So any fixes Zebra performs won't change user-specified data.
+#[derive(Copy, Clone, Debug)]
+pub struct FixHttpRequestMiddleware;
+
+impl RequestMiddleware for FixHttpRequestMiddleware {
+    fn on_request(
+        &self,
+        request: hyper::Request<hyper::Body>,
+    ) -> jsonrpc_http_server::RequestMiddlewareAction {
+        let request = request.map(|body| {
+            let body = body.map_ok(|data| {
+                // To simplify data handling, we assume that any search strings won't be split
+                // across multiple `Bytes` data buffers.
+                //
+                // To simplify error handling, Zebra only supports valid UTF-8 requests,
+                // and uses lossy UTF-8 conversion.
+                //
+                // JSON-RPC requires all requests to be valid UTF-8.
+                // The lower layers should reject invalid requests with lossy changes.
+                // But if they accept some lossy changes, that's ok,
+                // because the request was non-standard anyway.
+                //
+                // We're not concerned about performance here, so we just clone the Cow<str>
+                let data = String::from_utf8_lossy(data.as_ref()).to_string();
+
+                // Fix up the request.
+                let data = Self::remove_json_1_fields(data);
+
+                Bytes::from(data)
+            });
+
+            Body::wrap_stream(body)
+        });
+
+        jsonrpc_http_server::RequestMiddlewareAction::Proceed {
+            // TODO: disable this security check if we see errors from lightwalletd.
+            should_continue_on_invalid_cors: false,
+            request,
+        }
+    }
+}
+
+impl FixHttpRequestMiddleware {
+    /// Remove any "jsonrpc: 1.0" fields in `data`, and return the resulting string.
+    pub fn remove_json_1_fields(data: String) -> String {
+        // Replace "jsonrpc = 1.0":
+        // - at the start or middle of a list, and
+        // - at the end of a list;
+        // with no spaces (lightwalletd format), and spaces after separators (example format).
+        //
+        // TODO: if we see errors from lightwalletd, make this replacement more accurate:
+        //     - use a partial JSON fragment parser
+        //     - combine the whole request into a single buffer, and use a JSON parser
+        //     - use a regular expression
+        //
+        // We could also just handle the exact lightwalletd format,
+        // by replacing `{"jsonrpc":"1.0",` with `{`.
+        data.replace("\"jsonrpc\":\"1.0\",", "")
+            .replace("\"jsonrpc\": \"1.0\",", "")
+            .replace(",\"jsonrpc\":\"1.0\"", "")
+            .replace(", \"jsonrpc\": \"1.0\"", "")
+    }
+}

zebrad/Cargo.toml

@@ -13,6 +13,7 @@ default-run = "zebrad"
 zebra-chain = { path = "../zebra-chain" }
 zebra-consensus = { path = "../zebra-consensus/" }
 zebra-network = { path = "../zebra-network" }
+zebra-rpc = { path = "../zebra-rpc" }
 zebra-state = { path = "../zebra-state" }
 
 abscissa_core = "0.5"

zebrad/src/commands/start.rs

@@ -72,6 +72,8 @@ use zebra_chain::{
 };
 use zebra_consensus::CheckpointList;
 
+use zebra_rpc::server::RpcServer;
+
 use crate::{
     components::{
         inbound::{self, InboundSetupData},
@@ -194,6 +196,8 @@ impl StartCmd {
                 .in_current_span(),
         );
 
+        let rpc_task_handle = RpcServer::spawn(config.rpc);
+
         info!("spawned initial Zebra tasks");
 
         // TODO: put tasks into an ongoing FuturesUnordered and a startup FuturesUnordered?
@@ -204,6 +208,7 @@ impl StartCmd {
         pin!(mempool_queue_checker_task_handle);
         pin!(tx_gossip_task_handle);
         pin!(progress_task_handle);
+        pin!(rpc_task_handle);
 
         // startup tasks
         let groth16_download_handle_fused = (&mut groth16_download_handle).fuse();
@@ -245,6 +250,13 @@ impl StartCmd {
                     Ok(())
                 }
 
+                rpc_result = &mut rpc_task_handle => {
+                    rpc_result
+                        .expect("unexpected panic in the rpc task");
+                    info!("rpc task exited");
+                    Ok(())
+                }
+
                 // Unlike other tasks, we expect the download task to finish while Zebra is running.
                 groth16_download_result = &mut groth16_download_handle_fused => {
                     groth16_download_result
@@ -277,6 +289,7 @@ impl StartCmd {
         mempool_crawler_task_handle.abort();
         mempool_queue_checker_task_handle.abort();
         tx_gossip_task_handle.abort();
+        rpc_task_handle.abort();
 
         // startup tasks
         groth16_download_handle.abort();

zebrad/src/config.rs

@@ -10,6 +10,7 @@ use serde::{Deserialize, Serialize};
 
 use zebra_consensus::Config as ConsensusSection;
 use zebra_network::Config as NetworkSection;
+use zebra_rpc::config::Config as RpcSection;
 use zebra_state::Config as StateSection;
 
 use crate::components::{mempool::Config as MempoolSection, sync};
@@ -42,6 +43,9 @@ pub struct ZebradConfig {
 
     /// Mempool configuration
     pub mempool: MempoolSection,
+
+    /// RPC configuration
+    pub rpc: RpcSection,
 }
 
 /// Tracing configuration section.